Пример #1
0
    def start(self, args=''):
        """
        Starts the process with the given arguments.

        :param args: additional arguments when invoking the child, appended
                     to any arguments specified to the initializer.
        :type args: string or list of strings
        :return: An :class:`~kaa.InProgress` object, finished with the exitcode
                 when the child process terminates (when the
                 :attr:`~signals.exited` signal is emitted).

        The Process is registered with a global supervisor which holds a strong
        reference to the Process object while the child process remains
        active. 

        .. warning::

           If :meth:`~kaa.InProgress.timeout` is called on the returned
           InProgress and the timeout occurs, the InProgress returned by
           :meth:`start` will be finished with a :class:`~kaa.TimeoutException`
           even though the child process isn't actually dead.  You can always
           test the :attr:`running` property, or use the
           :attr:`~signals.finished` signal, which doesn't emit until the child
           process is genuinely dead.
        """
        if self._child and self._state != Process.STATE_HUNG:
            raise IOError(errno.EEXIST, 'Child process has already been started')

        if not self._shell:
            cmd = self._normalize_cmd(self._cmd) + self._normalize_cmd(args)
        else:
            # If passing through the shell, user must provide cmd and args 
            # as strings.
            if not isinstance(self._cmd, basestring) or not isinstance(args, basestring):
                raise ValueError('Command and arguments must be strings when shell=True')
            cmd = self._cmd + ' ' + args

        if self._in_progress.finished:
            self._in_progress = InProgress()
        self._in_progress.signals['abort'].connect_weak(lambda exc: self.stop())
        self._exitcode = None
        supervisor.register(self)

        log.debug("Spawning: %s", cmd)
        self._child = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE, preexec_fn=self._child_preexec,
                                       close_fds=True, shell=self._shell)

        self._stdin.wrap(self._child.stdin, IO_WRITE)
        self._stdout.wrap(self._child.stdout, IO_READ)
        self._stderr.wrap(self._child.stderr, IO_READ)
        self._state = Process.STATE_RUNNING
        return self._in_progress
Пример #2
0
    def read(self):
        """
        Reads a chunk of data from the channel.
        
        :returns: An :class:`~kaa.InProgress` object. If the InProgress is
                  finished with the empty string, it means that no data 
                  was collected and the channel was closed (or the channel 
                  was already closed when read() was called).

        It is therefore possible to busy-loop by reading on a closed channel::

            while True:
                data = yield channel.read()
                # Or: channel.read().wait()

        So the return value of read() should be checked.  Alternatively,
        the :attr:`readable` property could be tested::

            while channel.readable:
                 data = yield process.read()

        """
        if self._read_queue.tell() > 0:
            s = self._read_queue.getvalue()
            self._clear_read_queue()
            return InProgress().finish(s)

        return self._async_read(self._read_signal)
Пример #3
0
    def readline(self):
        """
        Reads a line from the channel.
        
        The line delimiter is included in the string to avoid ambiguity.  If no
        delimiter is present then either the read queue became full or the
        channel was closed before a delimiter was received.

        :returns: An :class:`~kaa.InProgress` object. If the InProgress is
                  finished with the empty string, it means that no data 
                  was collected and the channel was closed (or the channel 
                  was already closed when readline() was called).

        Data from the channel is read and queued in until the delimiter (\\\\n by
        default, but may be changed by the :attr:`delimiter`
        property) is found.  If the read queue size exceeds the queue limit,
        then the InProgress returned here will be finished prematurely with
        whatever is in the read queue, and the read queue will be purged.

        This method may not be called when a callback is connected to the
        IOChannel's readline signal.  You must use either one approach or the
        other.
        """
        if self._is_readline_connected() and len(self._readline_signal) == 0:
            # Connecting to 'readline' signal _and_ calling readline() is
            # not supported.  It's unclear how to behave in this case.
            raise RuntimeError('Callback currently connected to readline signal')

        line = self._pop_line_from_read_queue()
        if line:
            return InProgress().finish(line)
        return self._async_read(self._readline_signal)
Пример #4
0
    def write(self, data):
        """
        Writes the given data to the channel.
        
        :param data: the data to be written to the channel.
        :type data: string

        :returns: An :class:`~kaa.InProgress` object which is finished when the
                  given data is fully written to the channel.  The InProgress
                  is finished with the number of bytes sent in the last write 
                  required to commit the given data to the channel.  (This may
                  not be the actual number of bytes of the given data.)

                  If the channel closes unexpectedly before the data was
                  written, an IOError is thrown to the InProgress.

        It is not required that the channel be open in order to write to it.
        Written data is queued until the channel open and then flushed.  As
        writes are asynchronous, all written data is queued.  It is the
        caller's responsibility to ensure the internal write queue does not
        exceed the desired size by waiting for past write() InProgress to
        finish before writing more data.

        If a write does not complete because the channel was closed
        prematurely, an IOError is thrown to the InProgress.
        """
        if not (self._mode & IO_WRITE):
            raise IOError(9, 'Cannot write to a read-only channel')
        if not self.writable:
            raise IOError(9, 'Channel is not writable')
        if self.write_queue_used + len(data) > self._queue_size:
            raise ValueError('Data would exceed write queue limit')
        if not isinstance(data, BYTES_TYPE):
            raise ValueError('data must be bytes, not unicode')

        ip = InProgress()
        if data:
            ip.signals['abort'].connect(self._abort_write_inprogress, data, ip)
            self._write_queue.append((data, ip))
            if self._channel and self._wmon and not self._wmon.active:
                self._wmon.register(self.fileno, IO_WRITE)
        else:
            # We're writing the null string, nothing really to do.  We're
            # implicitly done.
            ip.finish(0)
        return ip
Пример #5
0
    def __call__(self, *args, **kwargs):
        in_progress = InProgress()

        if CoreThreading.is_mainthread():
            try:
                result = super(MainThreadCallable, self).__call__(*args, **kwargs)
            except BaseException, e:
                # All exceptions, including SystemExit and KeyboardInterrupt,
                # are caught and thrown to the InProgress, because it may be
                # waiting in another thread.  However SE and KI are reraised
                # in here the main thread so they can be propagated back up
                # the mainloop.
                in_progress.throw(*sys.exc_info())
                if isinstance(e, (KeyboardInterrupt, SystemExit)):
                    raise
            else:
                in_progress.finish(result)

            return in_progress
Пример #6
0
    def start(self, args=''):
        """
        Starts the process with the given arguments.

        :param args: additional arguments when invoking the child, appended
                     to any arguments specified to the initializer.
        :type args: string or list of strings
        :return: An :class:`~kaa.InProgress` object, finished with the exitcode
                 when the child process terminates (when the
                 :attr:`~signals.exited` signal is emitted).

        The Process is registered with a global supervisor which holds a strong
        reference to the Process object while the child process remains
        active. 

        .. warning::

           If :meth:`~kaa.InProgress.timeout` is called on the returned
           InProgress and the timeout occurs, the InProgress returned by
           :meth:`start` will be finished with a :class:`~kaa.TimeoutException`
           even though the child process isn't actually dead.  You can always
           test the :attr:`running` property, or use the
           :attr:`~signals.finished` signal, which doesn't emit until the child
           process is genuinely dead.
        """
        if self._child and self._state != Process.STATE_HUNG:
            raise IOError(errno.EEXIST, 'Child process has already been started')

        if not self._shell:
            cmd = self._normalize_cmd(self._cmd) + self._normalize_cmd(args)
        else:
            # If passing through the shell, user must provide cmd and args 
            # as strings.
            if not isinstance(self._cmd, basestring) or not isinstance(args, basestring):
                raise ValueError('Command and arguments must be strings when shell=True')
            cmd = self._cmd + ' ' + args

        if self._in_progress.finished:
            self._in_progress = InProgress()
        self._in_progress.signals['abort'].connect_weak(lambda exc: self.stop())
        self._exitcode = None
        supervisor.register(self)

        log.debug("Spawning: %s", cmd)
        self._child = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE, preexec_fn=self._child_preexec,
                                       close_fds=True, shell=self._shell)

        self._stdin.wrap(self._child.stdin, IO_WRITE)
        self._stdout.wrap(self._child.stdout, IO_READ)
        self._stderr.wrap(self._child.stderr, IO_READ)
        self._state = Process.STATE_RUNNING
        return self._in_progress
Пример #7
0
    def _async_read(self, stdout_read, stderr_read):
        """
        Common implementation for read() and readline().
        """
        if not self._stdout.readable and not self._stderr.readable:
            return InProgress().finish(None)

        # TODO: if child is dead, attach handler to this IP and if len data <
        # chunk size, can close the channel.  (What makes this more complicated
        # is knowing which channel to close, given pass_index=False.)
        return InProgressAny(stdout_read(), stderr_read(), pass_index=False,
                             filter=lambda val: val in (None, ''))
Пример #8
0
    def _async_read(self, signal):
        """
        Common implementation for read() and readline().
        """
        if not (self._mode & IO_READ):
            raise IOError(9, 'Cannot read on a write-only channel')
        if not self.readable:
            # channel is not readable.  Return an InProgress pre-finished
            # with None
            return InProgress().finish(None)

        ip = inprogress(signal)
        ip.signals['abort'].connect(self._abort_read_inprogress, signal, ip)
        return ip
Пример #9
0
    def __init__(self, function, function_info, interval, progress=None):
        InProgress.__init__(self)
        self._coroutine = function
        self._coroutine_info = function_info
        self._timer = Timer(self._step)
        self._interval = interval
        self._prerequisite_ip = None
        self._valid = True

        # This object (self) represents a coroutine that is in progress: that
        # is, at some point in the coroutine, it has yielded and expects
        # to be reentered at some point.  Even if there are no outside
        # references to this CoroutineInProgress object, the coroutine must
        # resume.
        #
        # Here, an "outside reference" refers to a reference kept by the
        # caller of the API (that is, not refs kept by kaa internally).
        #
        # For other types of InProgress objects, when there are no outside
        # references to them, clearly nobody is interested in the result, so
        # they can be destroyed.  For CoroutineInProgress, we mustn't rely
        # on outside references to keep the coroutine alive, so we keep refs
        # for active CoroutineInProgress objects in a global set called
        # _active_coroutines.  We then then remove ourselves from this set when
        # stopped.
        #
        _active_coroutines.add(self)

        if progress is NotFinished:
            # coroutine was stopped NotFinished, start the step timer
            self._timer.start(interval)
        elif isinstance(progress, InProgress):
            # continue when InProgress is done
            self._prerequisite_ip = progress
            progress.connect_both(self._continue, self._continue)
        elif progress is not None:
            raise AttributeError('invalid progress %s' % progress)
Пример #10
0
    def __call__(self, *args, **kwargs):
        in_progress = InProgress()

        if CoreThreading.is_mainthread():
            try:
                result = super(MainThreadCallable, self).__call__(*args, **kwargs)
            except BaseException, e:
                # All exceptions, including SystemExit and KeyboardInterrupt,
                # are caught and thrown to the InProgress, because it may be
                # waiting in another thread.  However SE and KI are reraised
                # in here the main thread so they can be propagated back up
                # the mainloop.
                in_progress.throw()
                if isinstance(e, (KeyboardInterrupt, SystemExit)):
                    raise
            else:
                in_progress.finish(result)

            return in_progress
Пример #11
0
    def __init__(self, cmd, shell=False, dumpfile=None):
        """
        Create a Process object.  The subprocess is not started until
        :meth:`start` is called.

        :param cmd: the command to be executed.
        :type cmd: string or list of strings
        :param shell: True if the command should be executed through a shell.
                      This allows for shell-like syntax (redirection, pipes,
                      etc.), but in this case *cmd* must be a string.
        :type shell: bool
        :param dumpfile: File to which all child stdout and stderr will be
                         dumped, or None to disable output dumping.
        :type dumpfile: None, string (path to filename), file object, IOChannel

        Process objects passed to :func:`kaa.inprogress` return a
        :class:`~kaa.InProgress` that corresponds to the
        :attr:`~kaa.Process.signals.exited` signal (not the
        :attr:`~kaa.Process.signals.finished` signal).
        """
        super(Process, self).__init__()
        self._cmd = cmd
        self._shell = shell
        self._stop_command = None
        # The subprocess.Popen object.
        self._child = None
        # Weakref of self used to invoke Process._cleanup callback on finalization.
        self._cleanup_weakref = None
        # The exit code returned by the child once it completes.
        self._exitcode = None

        if dumpfile:
            # Dumpfile specified, create IOChannel which we'll later pass to
            # IOSubChannels.  dumpfile can be a string (path to file), or
            # anything else you can pass to IOChannel (fd, file-like object,
            # another IOChannel, etc.)
            if isinstance(dumpfile, basestring):
                try:
                    dumpfile = open(dumpfile, 'w')
                    log.info('Logging process activity to %s' % dumpfile.name)
                except IOError:
                    log.warning('Unable to open %s for logging' % dumpfile)
            logger = IOChannel(dumpfile, mode=IO_WRITE)
        else:
            logger = None

        # Create the IOChannels for the child's stdin, stdout, and stderr.
        self._stdin = IOChannel()
        self._stdout = IOSubChannel(self, logger)
        self._stderr = IOSubChannel(self, logger)
        self._weak_closed_cbs = []

        for fd in self._stdout, self._stderr:
            fd.signals['read'].connect_weak(self.signals['read'].emit)
            fd.signals['readline'].connect_weak(self.signals['readline'].emit)
            # We need to keep track of the WeakCallables for _cleanup()
            cb = fd.signals['closed'].connect_weak(self._check_dead)
            self._weak_closed_cbs.append(cb)
        self._stdin.signals['closed'].connect_weak(self._check_dead)

        # The Process read and readline signals (aka "global" read/readline signals)
        # encapsulate both stdout and stderr.  When a new callback is connected
        # to these signals, we invoke _update_read_monitor() on the IOSubChannel
        # object which will register the fd with the mainloop if necessary.
        # (If we didn't do this, the fd would not get registered and therefore
        # data never read and therefore the callbacks connected to the global
        # read/readline signals never invoked.)
        cb = WeakCallable(self._update_read_monitor)
        self.signals['read'].changed_cb = cb
        self.signals['readline'].changed_cb = cb

        self._state = Process.STATE_STOPPED 
        # InProgress for the whole process.  Is recreated in start() for
        # multiple invocations, and finished when the process is terminated.
        self._in_progress = InProgress()
Пример #12
0
class Process(Object):

    STATE_STOPPED = 0  # Idle state, no child.
    STATE_RUNNING = 1  # start() was called and child is running
    STATE_STOPPING = 2 # stop() was called
    STATE_DYING = 3    # in the midst of cleanup during child death
    STATE_HUNG = 4     # a SIGKILL failed to stop the process

    __kaasignals__ = {
        'read':
            """
            Emitted for each chunk of data read from either stdout or stderr
            of the child process.

            .. describe:: def callback(chunk, ...)

               :param chunk: data read from the child's stdout or stderr.
               :type chunk: str

            When a callback is connected to the *read* signal, data is
            automatically read from the child as soon as it becomes available,
            and the signal is emitted.

            It is allowed to have a callback connected to the *read* signal and
            simultaneously use the :meth:`read` and :meth:`readline` methods.
            """,

        'readline':
            """
            Emitted for each line read from either stdout or stderr of the
            child process.

            .. describe:: def callback(line, ...)

               :param line: line read from the child's stdout or stderr.
               :type line: str

            It is not allowed to have a callback connected to the *readline* signal
            and simultaneously use the :meth:`readline` method.

            Refer to :meth:`readline` for more details.
            """,

        'finished':
            """
            Emitted when the child is dead and all data from stdout and stderr
            has been consumed.

            .. describe:: def callback(exitcode, ...)

               :param exitcode: the exit code of the child
               :type expected: int

            Due to buffering, a child process may be terminated, but the pipes
            to its stdout and stderr still open possibly containing buffered
            data yet to be read.  This signal emits only when the child has
            exited and all data has been consumed (or stdout and stderr
            explicitly closed).

            After this signal emits, the :attr:`readable` property will be
            False.
            """,

        'exited':
            """
            Emitted when the child process has terminated.

            .. describe:: def callback(exitcode, ...)

               :param exitcode: the exit code of the child
               :type expected: int

            Unlike the :attr:`~ksignals.finished` signal, this signal emits
            when the child is dead (and has been reaped), however the Process
            may or may not still be :attr:`readable`.
            """
    }

    
    def __init__(self, cmd, shell=False, dumpfile=None):
        """
        Create a Process object.  The subprocess is not started until
        :meth:`start` is called.

        :param cmd: the command to be executed.
        :type cmd: string or list of strings
        :param shell: True if the command should be executed through a shell.
                      This allows for shell-like syntax (redirection, pipes,
                      etc.), but in this case *cmd* must be a string.
        :type shell: bool
        :param dumpfile: File to which all child stdout and stderr will be
                         dumped, or None to disable output dumping.
        :type dumpfile: None, string (path to filename), file object, IOChannel

        Process objects passed to :func:`kaa.inprogress` return a
        :class:`~kaa.InProgress` that corresponds to the
        :attr:`~kaa.Process.signals.exited` signal (not the
        :attr:`~kaa.Process.signals.finished` signal).
        """
        super(Process, self).__init__()
        self._cmd = cmd
        self._shell = shell
        self._stop_command = None
        # The subprocess.Popen object.
        self._child = None
        # Weakref of self used to invoke Process._cleanup callback on finalization.
        self._cleanup_weakref = None
        # The exit code returned by the child once it completes.
        self._exitcode = None

        if dumpfile:
            # Dumpfile specified, create IOChannel which we'll later pass to
            # IOSubChannels.  dumpfile can be a string (path to file), or
            # anything else you can pass to IOChannel (fd, file-like object,
            # another IOChannel, etc.)
            if isinstance(dumpfile, basestring):
                try:
                    dumpfile = open(dumpfile, 'w')
                    log.info('Logging process activity to %s' % dumpfile.name)
                except IOError:
                    log.warning('Unable to open %s for logging' % dumpfile)
            logger = IOChannel(dumpfile, mode=IO_WRITE)
        else:
            logger = None

        # Create the IOChannels for the child's stdin, stdout, and stderr.
        self._stdin = IOChannel()
        self._stdout = IOSubChannel(self, logger)
        self._stderr = IOSubChannel(self, logger)
        self._weak_closed_cbs = []

        for fd in self._stdout, self._stderr:
            fd.signals['read'].connect_weak(self.signals['read'].emit)
            fd.signals['readline'].connect_weak(self.signals['readline'].emit)
            # We need to keep track of the WeakCallables for _cleanup()
            cb = fd.signals['closed'].connect_weak(self._check_dead)
            self._weak_closed_cbs.append(cb)
        self._stdin.signals['closed'].connect_weak(self._check_dead)

        # The Process read and readline signals (aka "global" read/readline signals)
        # encapsulate both stdout and stderr.  When a new callback is connected
        # to these signals, we invoke _update_read_monitor() on the IOSubChannel
        # object which will register the fd with the mainloop if necessary.
        # (If we didn't do this, the fd would not get registered and therefore
        # data never read and therefore the callbacks connected to the global
        # read/readline signals never invoked.)
        cb = WeakCallable(self._update_read_monitor)
        self.signals['read'].changed_cb = cb
        self.signals['readline'].changed_cb = cb

        self._state = Process.STATE_STOPPED 
        # InProgress for the whole process.  Is recreated in start() for
        # multiple invocations, and finished when the process is terminated.
        self._in_progress = InProgress()
        

    def _update_read_monitor(self, signal=None, change=None):
        """
        See IOChannel._update_read_monitor for docstring.
        """
        self._stdout._update_read_monitor(signal, change)
        self._stderr._update_read_monitor(signal, change)


    def __inprogress__(self):
        return self._in_progress


    @property
    def stdin(self):
        """
        :class:`~kaa.IOChannel` of child process's stdin.
        
        This object is valid even when the child is not running.
        """
        return self._stdin


    @property
    def stdout(self):
        """
        :class:`~kaa.IOChannel` of child process's stdout.
        
        This object is valid even when the child is not running, although it is
        obviously not readable until the child is started.
        """
        return self._stdout


    @property
    def stderr(self):
        """
        :class:`~kaa.IOChannel` of child process's stderr.
        
        This object is valid even when the child is not running, although it is
        obviously not readable until the child is started.
        """
        return self._stderr


    @property
    def pid(self):
        """
        The child's pid when it is running (or stopping), or None when it is not.
        """
        if self._child:
            return self._child.pid

    @property
    def exitcode(self):
        """
        The child's exit code once it has terminated.
        
        If the child is still running or it has not yet been started, this
        value will be None.
        """
        return self._exitcode


    @property
    def running(self):
        """
        True if the child process is running.

        A child that is currently stopping is still considered running.  When
        the :attr:`running` property is False, it means :meth:`start`
        may safely be called.

        To test whether :meth:`read` or :meth:`write` may be called, use the
        :attr:`readable` and :attr:`writable` properties respectively.
        """
        return bool(self._child and self._state not in (Process.STATE_STOPPED, Process.STATE_HUNG))


    @property
    def stopping(self):
        """
        True if the child process is currently being shut down.

        True when :meth:`stop` was called and the process is not stopped yet.
        """
        return bool(self._child and self._state == Process.STATE_STOPPING)


    @property
    def readable(self):
        """
        True if it is possible to read data from the child.

        The child is readable if either the child's :attr:`stdout` or
        :attr:`stderr` channels are still open, or if they are both closed but
        a read call would succeed anyway due to data remaining in the read
        queue.
        
        This doesn't necessarily mean the child is still running: a terminated
        child may still be read from (there may be data buffered in its
        :attr:`stdout` or :attr:`stderr` channels).  Use the :attr:`running`
        property if you want to see if the child is still running.
        """
        return self._stdout.readable or self._stderr.readable


    @property
    def writable(self):
        """
        True if it is possible to write data to the child.

        If the child process is writable, :meth:`write` may safely be called.
        A child that is in the process of :attr:`stopping` is not writable.
        """
        return bool(self._child and self._state == Process.STATE_RUNNING)


    @property
    def stop_command(self):
        """
        Stop command for this process.
        
        The command can be either a callable or a string.  The command is
        invoked (if it is a callable) or the command is written to the child's
        stdin (if cmd is a string or unicode) when the process is being
        terminated with a call to stop().

        Shutdown handlers for the process should be set with this property.
        """
        return self._stop_command


    @stop_command.setter
    def stop_command(self, cmd):
        assert(callable(cmd) or type(cmd) in (str, unicode) or cmd == None)
        self._stop_command = cmd


    @property
    def delimiter(self):
        """
        String used to split data for use with :meth:`readline`.
        """
        # stdout and stderr are the same.
        return self._stdout.delimiter


    @delimiter.setter
    def delimiter(self, value):
        self._stdout.delimiter = value
        self._stderr.delimiter = value

        
    def _normalize_cmd(self, cmd):
        """
        Returns a list of arguments based on the given cmd.  If cmd is a list,
        empty strings and other zero values are removed and the list is
        returned.  If cmd is a string, it is converted to a list based on shell
        semantics.  

        e.g. program -a "bar baz" \"blah -> ['program', '-a', 'bar baz', '"blah']
        """
        if cmd and isinstance(cmd, basestring):
            return shlex.split(cmd)
        elif isinstance(cmd, (tuple, list)):
            return [ x for x in cmd if cmd ]
        elif not cmd:
            return []

    def _child_preexec(self):
        """
        Callback function for Popen object that gets invoked in the child after
        forking but prior to execing.
        """
        # Children will inherit any ignored signals, so before we exec, reset
        # any ignored signals to their defaults.
        for sig in range(1, signal.NSIG):
            if signal.getsignal(sig) == signal.SIG_IGN:
                signal.signal(sig, signal.SIG_DFL)


    #@threaded() <-- don't
    def start(self, args=''):
        """
        Starts the process with the given arguments.

        :param args: additional arguments when invoking the child, appended
                     to any arguments specified to the initializer.
        :type args: string or list of strings
        :return: An :class:`~kaa.InProgress` object, finished with the exitcode
                 when the child process terminates (when the
                 :attr:`~signals.exited` signal is emitted).

        The Process is registered with a global supervisor which holds a strong
        reference to the Process object while the child process remains
        active. 

        .. warning::

           If :meth:`~kaa.InProgress.timeout` is called on the returned
           InProgress and the timeout occurs, the InProgress returned by
           :meth:`start` will be finished with a :class:`~kaa.TimeoutException`
           even though the child process isn't actually dead.  You can always
           test the :attr:`running` property, or use the
           :attr:`~signals.finished` signal, which doesn't emit until the child
           process is genuinely dead.
        """
        if self._child and self._state != Process.STATE_HUNG:
            raise IOError(errno.EEXIST, 'Child process has already been started')

        if not self._shell:
            cmd = self._normalize_cmd(self._cmd) + self._normalize_cmd(args)
        else:
            # If passing through the shell, user must provide cmd and args 
            # as strings.
            if not isinstance(self._cmd, basestring) or not isinstance(args, basestring):
                raise ValueError('Command and arguments must be strings when shell=True')
            cmd = self._cmd + ' ' + args

        if self._in_progress.finished:
            self._in_progress = InProgress()
        self._in_progress.signals['abort'].connect_weak(lambda exc: self.stop())
        self._exitcode = None
        supervisor.register(self)

        log.debug("Spawning: %s", cmd)
        self._child = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE, preexec_fn=self._child_preexec,
                                       close_fds=True, shell=self._shell)

        self._stdin.wrap(self._child.stdin, IO_WRITE)
        self._stdout.wrap(self._child.stdout, IO_READ)
        self._stderr.wrap(self._child.stderr, IO_READ)
        self._state = Process.STATE_RUNNING
        return self._in_progress


    @coroutine(policy=POLICY_SINGLETON)
    def stop(self, cmd=None, wait=3.0):
        """
        Stops the child process.
        
        :param cmd: stop command used to attempt to terminate the child
                    gracefully; overrides the :attr:`stop_command` property if
                    specified.
        :type cmd: string or callable
        :param wait: number of seconds to wait between termination steps 
                     (see below).
        :type wait: float

        :returns: A :class:`~kaa.InProgress`, finished (with None) when the
                  child terminates.  If the child refuses to terminate (even
                  with a SIGKILL) an SystemError exception is thrown to the
                  InProgress.

        The child process is terminated using the following steps:

            1. The stop command is written (or invoked) if one is specified,
               and up to *wait* seconds is given for the child to terminate.
            2. A SIGTERM is issued to the child process, and, again, we wait up
               to *wait* seconds for the child to terminate.
            3. A SIGKILL is issued to the child process, and this time
               we wait up to *wait*\*2 seconds.

        If after step 3 the child is still not dead, a SystemError exception is
        thrown to the InProgress, as well to the InProgress returned by
        :meth:`start` and the :attr:`~signals.finished` signal will be emitted
        with the value None.
        """
        if self._state != Process.STATE_RUNNING:
            # Process is either stopping or dying (or hung).
            yield

        self._state = Process.STATE_STOPPING
        cmd = cmd or self._stop_command
        pid = self.pid  # See below why we save self.pid

        if cmd:
            log.debug('Stop command specified: %s, stdin=%d', cmd, self._stdin.alive)
            if callable(cmd):
                # XXX: should we allow coroutines for cmd and yield them?
                cmd()
            elif self._stdin.alive:
                # This does get buffered.  We could bypass the write queue
                # by calling self.stdin._write() directly, but maybe the
                # IOChannel isn't writable at this moment.
                self._stdin.write(cmd)

            yield InProgressAny(self._in_progress, delay(wait))
            # If we're here, child is either dead or we timed out.

        self._stdin.close(immediate=True)
        # Either no stop command specified or our stop attempt timed out.
        # Try a relatively polite SIGTERM, then SIGKILL.
        for sig, pause in ((15, wait), (9, wait * 2)):
            if self._state == Process.STATE_STOPPED:
                # And we're done.
                yield
            try:
                os.kill(pid, sig)
                # Here we yield on the 'exited' signal instead of
                # self._in_progress, because the InProgress could in fact be
                # finished due to a timeout, not because the process is
                # legitimately stopped, whereas the 'exited' signals truly is
                # only emitted when the child is dead.  (Also, because we
                # don't care about losing stdout/err data, otherwise we would
                # use 'finished')
                yield InProgressAny(self.signals['exited'], delay(pause))
            except OSError:
                # Process is dead after all.
                self._check_dead()
            except:
                log.exception("Some other error")

        # If state isn't STOPPED, make sure pid hasn't changed.  Because we yield
        # on the 'exited' signal above, it's possible for the user to connect a 
        # callback to 'exited' that restarts the child, which gets executed before
        # this coroutine resumes.  If our pid has changed, we know we died.  If the
        # pid is the same, we have a hung process.
        if self._state != Process.STATE_STOPPED and pid == self.pid:
            # Child refuses to die even after SIGKILL. :(
            self._state = Process.STATE_HUNG
            exc = SystemError('Child process (pid=%d) refuses to die even after SIGKILL' % pid)
            self._in_progress.throw(SystemError, exc, None)
            raise exc


    def _async_read(self, stdout_read, stderr_read):
        """
        Common implementation for read() and readline().
        """
        if not self._stdout.readable and not self._stderr.readable:
            return InProgress().finish(None)

        # TODO: if child is dead, attach handler to this IP and if len data <
        # chunk size, can close the channel.  (What makes this more complicated
        # is knowing which channel to close, given finish=FINISH_RESULT.)
        return InProgressAny(stdout_read(), stderr_read(), finish=FINISH_RESULT,
                             filter=lambda val: val in (None, ''))


    def read(self):
        """
        Reads a chunk of data from either stdout or stderr of the process.

        There is no way to determine from which (stdout or stderr) the data
        was read; if you require this, use the :attr:`stdout` or :attr:`stderr`
        attributes directly (however see warning below).

        :returns: A :class:`~kaa.InProgress`, finished with the data read.
                  If it is finished the empty string, it means the child's
                  stdout and stderr were both closed (which is almost certainly
                  because the process exited) and no data was available.

        No exception is raised if the child is not readable.  Like
        :meth:`Socket.read`, it is therefore possible to busy-loop by reading
        on a dead child::

            while True:
                data = yield process.read()
                # Or: data = process.read().wait()

        So the return value of read() should be tested for non-None.
        Alternatively, the :attr:`readable` property could be tested::

            while process.readable:
                data = yield process.read()


        .. warning::
           You can read directly from stdout or stderr.  However, beware of this
           code, which is wrong::

               while process.readable:
                   data = yield process.stdout.read()

           In the above incorrect example, process.readable may be True even
           though process.stdout is closed (because process.stderr may not be
           closed).  In this case, process.stdout.read() will finish immediately
           with None, resulting in a busy loop.  The solution is to test the
           process.stdout.readable property instead::

               while process.stdout.readable:
                   data = yield process.stdout.read()
        """
        return self._async_read(self._stdout.read, self._stderr.read)


    def readline(self):
        """
        Reads a line from either stdout or stderr, whichever is available
        first.
        
        If finished with None or the empty string, it means that no data was
        read and the process exited.

        :returns: A :class:`~kaa.InProgress`, finished with the data read.
                  If it is finished the empty string, it means the child's
                  stdout and stderr were both closed (which is almost certainly
                  because the process exited) and no data was available.

        Like :meth:`read`, it is possible busy-loop with this method, so you
        should test its output or test the :attr:`readable` property calling.
        """
        return self._async_read(self._stdout.readline, self._stderr.readline)


    def write(self, data):
        """
        Write data to child's stdin.
        
        Returns an InProgress, which is finished when the data has actually
        been written to the child's stdin.

        :param data: the data to be written to the channel.
        :type data: string

        :returns: An :class:`~kaa.InProgress` object, which is finished when the
                  data has actually been written to the child's stdin.

                  If the channel closes unexpectedly before the data was
                  written, an IOError is thrown to the InProgress.

        This is a convenience function, as the caller could do
        ``process.stdin.write()``.
        """
        if not self._stdin.alive:
            raise IOError(9, 'Cannot write to closed child stdin')
        return self._stdin.write(data)


    @coroutine()
    def communicate(self, input=None):
        """
        One-time interaction with the process, sending the given input, and
        receiving all output from the child.

        :param input: the data to send to the child's stdin
        :type input: str
        :return: an :class:`~kaa.InProgress`, which will be finished with a
                 2-tuple (stdoutdata, stderrdata)

        If the process has not yet been started, :meth:`start` will be
        called implicitly.

        Any data previously written to the child with :meth:`write` will be
        flushed and the pipe to the child's stdin will be closed.  All
        subsequent data from the child's stdout and stderr will be read until
        EOF.  The child will be terminated before returning.

        This method is modeled after Python's standard library call
        ``subprocess.Popen.communicate()``.
        """
        if self._state == Process.STATE_STOPPED:
            self.start()

        try:
            if input:
                yield self.write(input)
            self.stdin.close()

            buf_out = BytesIO()
            while self.stdout.readable:
                buf_out.write((yield self.stdout.read()))

            buf_err = BytesIO()
            while self.stderr.readable:
                buf_err.write((yield self.stderr.read()))
        except InProgressAborted, e:
            # If the coroutine is aborted while we're trying to read from the
            # child's stdout/err, then stop the child.  We can't yield stop()
            # since aborted coroutines can't yield values.
            self.stop()
        else:
Пример #13
0
 def __init__(self):
     self._waiting = None
     self._finished = []
     self._generator_exit = False
     self._populate = InProgress()
Пример #14
0
 def __init__(self, callback, *args, **kwargs):
     InProgress.__init__(self)
     self._callback = Callback(callback, *args, **kwargs)
Пример #15
0
class Process(Object):

    STATE_STOPPED = 0  # Idle state, no child.
    STATE_RUNNING = 1  # start() was called and child is running
    STATE_STOPPING = 2  # stop() was called
    STATE_DYING = 3  # in the midst of cleanup during child death
    STATE_HUNG = 4  # a SIGKILL failed to stop the process

    __kaasignals__ = {
        'read':
        """
            Emitted for each chunk of data read from either stdout or stderr
            of the child process.

            .. describe:: def callback(chunk, ...)

               :param chunk: data read from the child's stdout or stderr.
               :type chunk: str

            When a callback is connected to the *read* signal, data is
            automatically read from the child as soon as it becomes available,
            and the signal is emitted.

            It is allowed to have a callback connected to the *read* signal and
            simultaneously use the :meth:`read` and :meth:`readline` methods.
            """,
        'readline':
        """
            Emitted for each line read from either stdout or stderr of the
            child process.

            .. describe:: def callback(line, ...)

               :param line: line read from the child's stdout or stderr.
               :type line: str

            It is not allowed to have a callback connected to the *readline* signal
            and simultaneously use the :meth:`readline` method.

            Refer to :meth:`readline` for more details.
            """,
        'finished':
        """
            Emitted when the child is dead and all data from stdout and stderr
            has been consumed.

            .. describe:: def callback(exitcode, ...)

               :param exitcode: the exit code of the child
               :type expected: int

            Due to buffering, a child process may be terminated, but the pipes
            to its stdout and stderr still open possibly containing buffered
            data yet to be read.  This signal emits only when the child has
            exited and all data has been consumed (or stdout and stderr
            explicitly closed).

            After this signal emits, the :attr:`readable` property will be
            False.
            """,
        'exited':
        """
            Emitted when the child process has terminated.

            .. describe:: def callback(exitcode, ...)

               :param exitcode: the exit code of the child
               :type expected: int

            Unlike the :attr:`~ksignals.finished` signal, this signal emits
            when the child is dead (and has been reaped), however the Process
            may or may not still be :attr:`readable`.
            """
    }

    def __init__(self, cmd, shell=False, dumpfile=None):
        """
        Create a Process object.  The subprocess is not started until
        :meth:`start` is called.

        :param cmd: the command to be executed.
        :type cmd: string or list of strings
        :param shell: True if the command should be executed through a shell.
                      This allows for shell-like syntax (redirection, pipes,
                      etc.), but in this case *cmd* must be a string.
        :type shell: bool
        :param dumpfile: File to which all child stdout and stderr will be
                         dumped, or None to disable output dumping.
        :type dumpfile: None, string (path to filename), file object, IOChannel

        Process objects passed to :func:`kaa.inprogress` return a
        :class:`~kaa.InProgress` that corresponds to the
        :attr:`~kaa.Process.signals.exited` signal (not the
        :attr:`~kaa.Process.signals.finished` signal).
        """
        super(Process, self).__init__()
        self._cmd = cmd
        self._shell = shell
        self._stop_command = None
        # The subprocess.Popen object.
        self._child = None
        # Weakref of self used to invoke Process._cleanup callback on finalization.
        self._cleanup_weakref = None
        # The exit code returned by the child once it completes.
        self._exitcode = None

        if dumpfile:
            # Dumpfile specified, create IOChannel which we'll later pass to
            # IOSubChannels.  dumpfile can be a string (path to file), or
            # anything else you can pass to IOChannel (fd, file-like object,
            # another IOChannel, etc.)
            if isinstance(dumpfile, basestring):
                try:
                    dumpfile = open(dumpfile, 'w')
                    log.info('Logging process activity to %s' % dumpfile.name)
                except IOError:
                    log.warning('Unable to open %s for logging' % dumpfile)
            logger = IOChannel(dumpfile, mode=IO_WRITE)
        else:
            logger = None

        # Create the IOChannels for the child's stdin, stdout, and stderr.
        self._stdin = IOChannel()
        self._stdout = IOSubChannel(self, logger)
        self._stderr = IOSubChannel(self, logger)
        self._weak_closed_cbs = []

        for fd in self._stdout, self._stderr:
            fd.signals['read'].connect_weak(self.signals['read'].emit)
            fd.signals['readline'].connect_weak(self.signals['readline'].emit)
            # We need to keep track of the WeakCallables for _cleanup()
            cb = fd.signals['closed'].connect_weak(self._check_dead)
            self._weak_closed_cbs.append(cb)
        self._stdin.signals['closed'].connect_weak(self._check_dead)

        # The Process read and readline signals (aka "global" read/readline signals)
        # encapsulate both stdout and stderr.  When a new callback is connected
        # to these signals, we invoke _update_read_monitor() on the IOSubChannel
        # object which will register the fd with the mainloop if necessary.
        # (If we didn't do this, the fd would not get registered and therefore
        # data never read and therefore the callbacks connected to the global
        # read/readline signals never invoked.)
        cb = WeakCallable(self._update_read_monitor)
        self.signals['read'].changed_cb = cb
        self.signals['readline'].changed_cb = cb

        self._state = Process.STATE_STOPPED
        # InProgress for the whole process.  Is recreated in start() for
        # multiple invocations, and finished when the process is terminated.
        self._in_progress = InProgress()

    def _update_read_monitor(self, signal=None, change=None):
        """
        See IOChannel._update_read_monitor for docstring.
        """
        self._stdout._update_read_monitor(signal, change)
        self._stderr._update_read_monitor(signal, change)

    def __inprogress__(self):
        return self._in_progress

    @property
    def stdin(self):
        """
        :class:`~kaa.IOChannel` of child process's stdin.
        
        This object is valid even when the child is not running.
        """
        return self._stdin

    @property
    def stdout(self):
        """
        :class:`~kaa.IOChannel` of child process's stdout.
        
        This object is valid even when the child is not running, although it is
        obviously not readable until the child is started.
        """
        return self._stdout

    @property
    def stderr(self):
        """
        :class:`~kaa.IOChannel` of child process's stderr.
        
        This object is valid even when the child is not running, although it is
        obviously not readable until the child is started.
        """
        return self._stderr

    @property
    def pid(self):
        """
        The child's pid when it is running (or stopping), or None when it is not.
        """
        if self._child:
            return self._child.pid

    @property
    def exitcode(self):
        """
        The child's exit code once it has terminated.
        
        If the child is still running or it has not yet been started, this
        value will be None.
        """
        return self._exitcode

    @property
    def running(self):
        """
        True if the child process is running.

        A child that is currently stopping is still considered running.  When
        the :attr:`running` property is False, it means :meth:`start`
        may safely be called.

        To test whether :meth:`read` or :meth:`write` may be called, use the
        :attr:`readable` and :attr:`writable` properties respectively.
        """
        return bool(
            self._child
            and self._state not in (Process.STATE_STOPPED, Process.STATE_HUNG))

    @property
    def stopping(self):
        """
        True if the child process is currently being shut down.

        True when :meth:`stop` was called and the process is not stopped yet.
        """
        return bool(self._child and self._state == Process.STATE_STOPPING)

    @property
    def readable(self):
        """
        True if it is possible to read data from the child.

        The child is readable if either the child's :attr:`stdout` or
        :attr:`stderr` channels are still open, or if they are both closed but
        a read call would succeed anyway due to data remaining in the read
        queue.
        
        This doesn't necessarily mean the child is still running: a terminated
        child may still be read from (there may be data buffered in its
        :attr:`stdout` or :attr:`stderr` channels).  Use the :attr:`running`
        property if you want to see if the child is still running.
        """
        return self._stdout.readable or self._stderr.readable

    @property
    def writable(self):
        """
        True if it is possible to write data to the child.

        If the child process is writable, :meth:`write` may safely be called.
        A child that is in the process of :attr:`stopping` is not writable.
        """
        return bool(self._child and self._state == Process.STATE_RUNNING)

    @property
    def stop_command(self):
        """
        Stop command for this process.
        
        The command can be either a callable or a string.  The command is
        invoked (if it is a callable) or the command is written to the child's
        stdin (if cmd is a string or unicode) when the process is being
        terminated with a call to stop().

        Shutdown handlers for the process should be set with this property.
        """
        return self._stop_command

    @stop_command.setter
    def stop_command(self, cmd):
        assert (callable(cmd) or type(cmd) in (str, unicode) or cmd == None)
        self._stop_command = cmd

    @property
    def delimiter(self):
        """
        String used to split data for use with :meth:`readline`.
        """
        # stdout and stderr are the same.
        return self._stdout.delimiter

    @delimiter.setter
    def delimiter(self, value):
        self._stdout.delimiter = value
        self._stderr.delimiter = value

    def _normalize_cmd(self, cmd):
        """
        Returns a list of arguments based on the given cmd.  If cmd is a list,
        empty strings and other zero values are removed and the list is
        returned.  If cmd is a string, it is converted to a list based on shell
        semantics.  

        e.g. program -a "bar baz" \"blah -> ['program', '-a', 'bar baz', '"blah']
        """
        if cmd and isinstance(cmd, basestring):
            return shlex.split(cmd)
        elif isinstance(cmd, (tuple, list)):
            return [x for x in cmd if cmd]
        elif not cmd:
            return []

    def _child_preexec(self):
        """
        Callback function for Popen object that gets invoked in the child after
        forking but prior to execing.
        """
        # Children will inherit any ignored signals, so before we exec, reset
        # any ignored signals to their defaults.
        for sig in range(1, signal.NSIG):
            if signal.getsignal(sig) == signal.SIG_IGN:
                signal.signal(sig, signal.SIG_DFL)

    #@threaded() <-- don't
    def start(self, args=''):
        """
        Starts the process with the given arguments.

        :param args: additional arguments when invoking the child, appended
                     to any arguments specified to the initializer.
        :type args: string or list of strings
        :return: An :class:`~kaa.InProgress` object, finished with the exitcode
                 when the child process terminates (when the
                 :attr:`~signals.exited` signal is emitted).

        The Process is registered with a global supervisor which holds a strong
        reference to the Process object while the child process remains
        active. 

        .. warning::

           If :meth:`~kaa.InProgress.timeout` is called on the returned
           InProgress and the timeout occurs, the InProgress returned by
           :meth:`start` will be finished with a :class:`~kaa.TimeoutException`
           even though the child process isn't actually dead.  You can always
           test the :attr:`running` property, or use the
           :attr:`~signals.finished` signal, which doesn't emit until the child
           process is genuinely dead.
        """
        if self._child and self._state != Process.STATE_HUNG:
            raise IOError(errno.EEXIST,
                          'Child process has already been started')

        if not self._shell:
            cmd = self._normalize_cmd(self._cmd) + self._normalize_cmd(args)
        else:
            # If passing through the shell, user must provide cmd and args
            # as strings.
            if not isinstance(self._cmd, basestring) or not isinstance(
                    args, basestring):
                raise ValueError(
                    'Command and arguments must be strings when shell=True')
            cmd = self._cmd + ' ' + args

        if self._in_progress.finished:
            self._in_progress = InProgress()
        self._in_progress.signals['abort'].connect_weak(
            lambda exc: self.stop())
        self._exitcode = None
        supervisor.register(self)

        log.debug("Spawning: %s", cmd)
        self._child = subprocess.Popen(cmd,
                                       stdin=subprocess.PIPE,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE,
                                       preexec_fn=self._child_preexec,
                                       close_fds=True,
                                       shell=self._shell)

        self._stdin.wrap(self._child.stdin, IO_WRITE)
        self._stdout.wrap(self._child.stdout, IO_READ)
        self._stderr.wrap(self._child.stderr, IO_READ)
        self._state = Process.STATE_RUNNING
        return self._in_progress

    @coroutine(policy=POLICY_SINGLETON)
    def stop(self, cmd=None, wait=3.0):
        """
        Stops the child process.
        
        :param cmd: stop command used to attempt to terminate the child
                    gracefully; overrides the :attr:`stop_command` property if
                    specified.
        :type cmd: string or callable
        :param wait: number of seconds to wait between termination steps 
                     (see below).
        :type wait: float

        :returns: A :class:`~kaa.InProgress`, finished (with None) when the
                  child terminates.  If the child refuses to terminate (even
                  with a SIGKILL) an SystemError exception is thrown to the
                  InProgress.

        The child process is terminated using the following steps:

            1. The stop command is written (or invoked) if one is specified,
               and up to *wait* seconds is given for the child to terminate.
            2. A SIGTERM is issued to the child process, and, again, we wait up
               to *wait* seconds for the child to terminate.
            3. A SIGKILL is issued to the child process, and this time
               we wait up to *wait*\*2 seconds.

        If after step 3 the child is still not dead, a SystemError exception is
        thrown to the InProgress, as well to the InProgress returned by
        :meth:`start` and the :attr:`~signals.finished` signal will be emitted
        with the value None.
        """
        if self._state != Process.STATE_RUNNING:
            # Process is either stopping or dying (or hung).
            yield

        self._state = Process.STATE_STOPPING
        cmd = cmd or self._stop_command
        pid = self.pid  # See below why we save self.pid

        if cmd:
            log.debug('Stop command specified: %s, stdin=%d', cmd,
                      self._stdin.alive)
            if callable(cmd):
                # XXX: should we allow coroutines for cmd and yield them?
                cmd()
            elif self._stdin.alive:
                # This does get buffered.  We could bypass the write queue
                # by calling self.stdin._write() directly, but maybe the
                # IOChannel isn't writable at this moment.
                self._stdin.write(cmd)

            yield InProgressAny(self._in_progress, delay(wait))
            # If we're here, child is either dead or we timed out.

        self._stdin.close(immediate=True)
        # Either no stop command specified or our stop attempt timed out.
        # Try a relatively polite SIGTERM, then SIGKILL.
        for sig, pause in ((15, wait), (9, wait * 2)):
            if self._state == Process.STATE_STOPPED:
                # And we're done.
                yield
            try:
                os.kill(pid, sig)
                # Here we yield on the 'exited' signal instead of
                # self._in_progress, because the InProgress could in fact be
                # finished due to a timeout, not because the process is
                # legitimately stopped, whereas the 'exited' signals truly is
                # only emitted when the child is dead.  (Also, because we
                # don't care about losing stdout/err data, otherwise we would
                # use 'finished')
                yield InProgressAny(self.signals['exited'], delay(pause))
            except OSError:
                # Process is dead after all.
                self._check_dead()
            except:
                log.exception("Some other error")

        # If state isn't STOPPED, make sure pid hasn't changed.  Because we yield
        # on the 'exited' signal above, it's possible for the user to connect a
        # callback to 'exited' that restarts the child, which gets executed before
        # this coroutine resumes.  If our pid has changed, we know we died.  If the
        # pid is the same, we have a hung process.
        if self._state != Process.STATE_STOPPED and pid == self.pid:
            # Child refuses to die even after SIGKILL. :(
            self._state = Process.STATE_HUNG
            exc = SystemError(
                'Child process (pid=%d) refuses to die even after SIGKILL' %
                pid)
            self._in_progress.throw(SystemError, exc, None)
            raise exc

    def _async_read(self, stdout_read, stderr_read):
        """
        Common implementation for read() and readline().
        """
        if not self._stdout.readable and not self._stderr.readable:
            return InProgress().finish(None)

        # TODO: if child is dead, attach handler to this IP and if len data <
        # chunk size, can close the channel.  (What makes this more complicated
        # is knowing which channel to close, given finish=FINISH_RESULT.)
        return InProgressAny(stdout_read(),
                             stderr_read(),
                             finish=FINISH_RESULT,
                             filter=lambda val: val in (None, ''))

    def read(self):
        """
        Reads a chunk of data from either stdout or stderr of the process.

        There is no way to determine from which (stdout or stderr) the data
        was read; if you require this, use the :attr:`stdout` or :attr:`stderr`
        attributes directly (however see warning below).

        :returns: A :class:`~kaa.InProgress`, finished with the data read.
                  If it is finished the empty string, it means the child's
                  stdout and stderr were both closed (which is almost certainly
                  because the process exited) and no data was available.

        No exception is raised if the child is not readable.  Like
        :meth:`Socket.read`, it is therefore possible to busy-loop by reading
        on a dead child::

            while True:
                data = yield process.read()
                # Or: data = process.read().wait()

        So the return value of read() should be tested for non-None.
        Alternatively, the :attr:`readable` property could be tested::

            while process.readable:
                data = yield process.read()


        .. warning::
           You can read directly from stdout or stderr.  However, beware of this
           code, which is wrong::

               while process.readable:
                   data = yield process.stdout.read()

           In the above incorrect example, process.readable may be True even
           though process.stdout is closed (because process.stderr may not be
           closed).  In this case, process.stdout.read() will finish immediately
           with None, resulting in a busy loop.  The solution is to test the
           process.stdout.readable property instead::

               while process.stdout.readable:
                   data = yield process.stdout.read()
        """
        return self._async_read(self._stdout.read, self._stderr.read)

    def readline(self):
        """
        Reads a line from either stdout or stderr, whichever is available
        first.
        
        If finished with None or the empty string, it means that no data was
        read and the process exited.

        :returns: A :class:`~kaa.InProgress`, finished with the data read.
                  If it is finished the empty string, it means the child's
                  stdout and stderr were both closed (which is almost certainly
                  because the process exited) and no data was available.

        Like :meth:`read`, it is possible busy-loop with this method, so you
        should test its output or test the :attr:`readable` property calling.
        """
        return self._async_read(self._stdout.readline, self._stderr.readline)

    def write(self, data):
        """
        Write data to child's stdin.
        
        Returns an InProgress, which is finished when the data has actually
        been written to the child's stdin.

        :param data: the data to be written to the channel.
        :type data: string

        :returns: An :class:`~kaa.InProgress` object, which is finished when the
                  data has actually been written to the child's stdin.

                  If the channel closes unexpectedly before the data was
                  written, an IOError is thrown to the InProgress.

        This is a convenience function, as the caller could do
        ``process.stdin.write()``.
        """
        if not self._stdin.alive:
            raise IOError(9, 'Cannot write to closed child stdin')
        return self._stdin.write(data)

    @coroutine()
    def communicate(self, input=None):
        """
        One-time interaction with the process, sending the given input, and
        receiving all output from the child.

        :param input: the data to send to the child's stdin
        :type input: str
        :return: an :class:`~kaa.InProgress`, which will be finished with a
                 2-tuple (stdoutdata, stderrdata)

        If the process has not yet been started, :meth:`start` will be
        called implicitly.

        Any data previously written to the child with :meth:`write` will be
        flushed and the pipe to the child's stdin will be closed.  All
        subsequent data from the child's stdout and stderr will be read until
        EOF.  The child will be terminated before returning.

        This method is modeled after Python's standard library call
        ``subprocess.Popen.communicate()``.
        """
        if self._state == Process.STATE_STOPPED:
            self.start()

        try:
            if input:
                yield self.write(input)
            self.stdin.close()

            buf_out = BytesIO()
            while self.stdout.readable:
                buf_out.write((yield self.stdout.read()))

            buf_err = BytesIO()
            while self.stderr.readable:
                buf_err.write((yield self.stderr.read()))
        except InProgressAborted, e:
            # If the coroutine is aborted while we're trying to read from the
            # child's stdout/err, then stop the child.  We can't yield stop()
            # since aborted coroutines can't yield values.
            self.stop()
        else:
Пример #16
0
    def __init__(self, cmd, shell=False, dumpfile=None):
        """
        Create a Process object.  The subprocess is not started until
        :meth:`start` is called.

        :param cmd: the command to be executed.
        :type cmd: string or list of strings
        :param shell: True if the command should be executed through a shell.
                      This allows for shell-like syntax (redirection, pipes,
                      etc.), but in this case *cmd* must be a string.
        :type shell: bool
        :param dumpfile: File to which all child stdout and stderr will be
                         dumped, or None to disable output dumping.
        :type dumpfile: None, string (path to filename), file object, IOChannel

        Process objects passed to :func:`kaa.inprogress` return a
        :class:`~kaa.InProgress` that corresponds to the
        :attr:`~kaa.Process.signals.exited` signal (not the
        :attr:`~kaa.Process.signals.finished` signal).
        """
        super(Process, self).__init__()
        self._cmd = cmd
        self._shell = shell
        self._stop_command = None
        # The subprocess.Popen object.
        self._child = None
        # Weakref of self used to invoke Process._cleanup callback on finalization.
        self._cleanup_weakref = None
        # The exit code returned by the child once it completes.
        self._exitcode = None

        if dumpfile:
            # Dumpfile specified, create IOChannel which we'll later pass to
            # IOSubChannels.  dumpfile can be a string (path to file), or
            # anything else you can pass to IOChannel (fd, file-like object,
            # another IOChannel, etc.)
            if isinstance(dumpfile, basestring):
                try:
                    dumpfile = open(dumpfile, 'w')
                    log.info('Logging process activity to %s' % dumpfile.name)
                except IOError:
                    log.warning('Unable to open %s for logging' % dumpfile)
            logger = IOChannel(dumpfile, mode=IO_WRITE)
        else:
            logger = None

        # Create the IOChannels for the child's stdin, stdout, and stderr.
        self._stdin = IOChannel()
        self._stdout = IOSubChannel(self, logger)
        self._stderr = IOSubChannel(self, logger)
        self._weak_closed_cbs = []

        for fd in self._stdout, self._stderr:
            fd.signals['read'].connect_weak(self.signals['read'].emit)
            fd.signals['readline'].connect_weak(self.signals['readline'].emit)
            # We need to keep track of the WeakCallables for _cleanup()
            cb = fd.signals['closed'].connect_weak(self._check_dead)
            self._weak_closed_cbs.append(cb)
        self._stdin.signals['closed'].connect_weak(self._check_dead)

        # The Process read and readline signals (aka "global" read/readline signals)
        # encapsulate both stdout and stderr.  When a new callback is connected
        # to these signals, we invoke _update_read_monitor() on the IOSubChannel
        # object which will register the fd with the mainloop if necessary.
        # (If we didn't do this, the fd would not get registered and therefore
        # data never read and therefore the callbacks connected to the global
        # read/readline signals never invoked.)
        cb = WeakCallable(self._update_read_monitor)
        self.signals['read'].changed_cb = cb
        self.signals['readline'].changed_cb = cb

        self._state = Process.STATE_STOPPED
        # InProgress for the whole process.  Is recreated in start() for
        # multiple invocations, and finished when the process is terminated.
        self._in_progress = InProgress()
Пример #17
0
class Generator(object):
    """
    Generator for InProgress objects
    """
    def __init__(self):
        self._waiting = None
        self._finished = []
        self._generator_exit = False
        self._populate = InProgress()

    def __inprogress__(self):
        """
        Wait until at least one item is produced or the generator
        finished before that happens.
        """
        return self._populate

    @threaded(MAINTHREAD)
    def send(self, result, exception=False):
        """
        Send a new value (producer)
        """
        delayed = [ InProgress(), result, False, exception ]
        if result is GeneratorExit:
            self._generator_exit = True
            delayed = None
        if not self._populate.finished:
            # First item
            self._delay = result
            self._waiting = delayed
            self._populate.finish(self)
            return
        ip, result, handled, exception = self._waiting
        if not handled:
            # InProgress not returned in __iter__, add it to the
            # finished objects
            self._finished.append(ip)
        self._waiting = delayed
        if exception:
            ip.throw(*result)
        else:
            ip.finish(result)

    def throw(self, type, value, tb):
        """
        Throw an error, this will stop the generator
        """
        self.send((type, value, tb), exception=True)
        self.send(GeneratorExit)
        return False

    def finish(self, result):
        """
        Finish the generator, the result will be ignored
        """
        self.send(GeneratorExit)

    def __iter__(self):
        """
        Iterate over the values (consumer)
        """
        while not self._generator_exit or self._waiting or self._finished:
            if not self._finished:
                # no finished items yet, return the waiting InProgress
                self._waiting[2] = True
                yield self._waiting[0]
            else:
                yield self._finished.pop(0)