Example #1
0
    def __init__(self, cmd, runas=None, pty=False, setpgrp=None, debug=False):
        """Args:
        'cmd' what command to execute
            Can be a list ("/bin/ls", "-la")
            or a string "/bin/ls" (will be passed to sh -c)

        'pty' do we allocate a pty for command?
        'runas' user we run as (set user, set groups, etc.)
        'setpgrp' do we setpgrp in child? (create its own process group)
        """

        self.ppid = os.getpid()

        self._child = None
        self._child = popen4.Popen4(cmd, 0, pty, runas, setpgrp)
        self.tochild = self._child.tochild
        self._fromchild = None

        self.pid = self._child.pid

        self._setpgrp = setpgrp
        self._debug = debug
        self._cmd = cmd

        self._output = FIFOBuffer()
        self._dprint("# command started (pid=%d, pty=%s): %s" %
                     (self._child.pid, repr(pty), cmd))
Example #2
0
class Command(object):
    """Convenience module for executing a command

    attribute notes::

        'exitcode' - None if the process hasn't exited, exitcode otherwise

        'output' - None if the process hasn't exited and fromchild hasn't
                   been accessed, the full output of the process

        'running' - True if process is still running, False otherwise

        'terminated' - Returns signal number if terminated, None otherwise

    Usage example::

        c = Command("./test.py")
        if c.running:
            c.wait()

        assert c.running is False

        print c.output

        c = Command("./test.py")

        # Unless you read from command.fromchild or use
        # command.outputsearch() command.output will be None until the
        # command finishes.

        while c.output is None: 
            time.sleep(1)

        print "output = '%s', exitcode = %d" % (c.output, c.exitcode)

        c = Command("cat", pty=True)
        print >> c.tochild, "test"
        print c.fromchild.readline(),

    """
    class Error(Exception):
        pass

    class _ChildObserver(Observer):
        def __init__(self, outputbuf, debug=False):
            self.debug = debug
            self.outputbuf = outputbuf

        def _dprint(self, event, msg):
            if self.debug:
                print("# EVENT '%s':\n%s" % (event, msg), file=sys.stderr)

        def notify(self, subject, event, val):
            if event in ('read', 'readline'):
                self._dprint(event, val)
                self.outputbuf.write(val)
            elif event in ('readlines', 'xreadlines'):
                self._dprint(event, "".join(val))
                self.outputbuf.write("".join(val))

    def __init__(self, cmd, runas=None, pty=False, setpgrp=None, debug=False):
        """Args:
        'cmd' what command to execute
            Can be a list ("/bin/ls", "-la")
            or a string "/bin/ls" (will be passed to sh -c)

        'pty' do we allocate a pty for command?
        'runas' user we run as (set user, set groups, etc.)
        'setpgrp' do we setpgrp in child? (create its own process group)
        """

        self.ppid = os.getpid()

        self._child = None
        self._child = popen4.Popen4(cmd, 0, pty, runas, setpgrp)
        self.tochild = self._child.tochild
        self._fromchild = None

        self.pid = self._child.pid

        self._setpgrp = setpgrp
        self._debug = debug
        self._cmd = cmd

        self._output = FIFOBuffer()
        self._dprint("# command started (pid=%d, pty=%s): %s" %
                     (self._child.pid, repr(pty), cmd))

    def __del__(self):
        if not self._child:
            return

        # don't terminate() a process we didn't start
        if os.getpid() == self.ppid:
            self.terminate()

    def _dprint(self, msg):
        if self._debug:
            print(msg, file=sys.stderr)

    def terminate(self, gracetime=0, sig=signal.SIGTERM):
        """terminate command. kills command with 'sig', then sleeps for 'gracetime', before sending SIGKILL
        """

        if self.running:
            if self._child.pty:
                cc_magic = termios.tcgetattr(self._child.tochild.fileno())[-1]
                ctrl_c = cc_magic[termios.VINTR]
                self._child.tochild.write(ctrl_c)

            pid = self.pid
            if self._setpgrp:
                pid = -pid

            try:
                os.kill(pid, sig)
            except OSError as e:
                if e[0] != errno.EPERM or \
                   not self._child.pty or \
                   not self.wait(timeout=6, poll_interval=0.1):
                    raise

                return

            for i in range(gracetime):
                if not self.running:
                    return
                time.sleep(1)

            if self.running:
                os.kill(pid, signal.SIGKILL)

                if not self.wait(timeout=3, poll_interval=0.1):
                    raise self.Error("process just won't die!")

                self._dprint("# command (pid %d) terminated" % self._child.pid)

    def terminated(self):
        status = self._child.poll()

        if not os.WIFSIGNALED(status):
            return None

        return os.WTERMSIG(status)

    terminated = property(terminated)

    def running(self):
        if self._child.poll() == -1:
            return True

        return False

    running = property(running)

    def exitcode(self):
        if self.running:
            return None

        status = self._child.poll()

        if not os.WIFEXITED(status):
            return None

        return os.WEXITSTATUS(status)

    exitcode = property(exitcode)

    def wait(self, timeout=None, poll_interval=0.2, callback=None):
        """wait for process to finish executing.
        'timeout' is how long we wait in seconds (None is forever)
        'poll_interval' is how long we sleep between checks to see if process has finished
        'callback': you can use callback to check for other conditions (e.g., besides timeout) and stop wait early.

        return value: did the process finish? True/False

        """
        if not self.running:
            return True

        if timeout is None:
            self._child.wait()
            return True
        else:
            start = time.time()
            while time.time() - start < timeout:
                if callback and callback() is False:
                    return False

                if not self.running:
                    return True
                time.sleep(poll_interval)

            return False

    def output(self):
        if len(self._output):
            return self._output.getvalue()

        if self.running:
            return None

        # this will read into self._output via _ChildObserver
        self.fromchild.read()

        return self._output.getvalue()

    output = property(output)

    def fromchild(self):
        """return the command's filehandler.

        NOTE: this file handler magically updates self._output"""

        if self._fromchild:
            return self._fromchild

        fh = FileEventAdaptor(self._child.fromchild)
        fh.addObserver(self._ChildObserver(self._output, self._debug))

        fh = FileEnhancedRead(fh)

        self._fromchild = fh
        return self._fromchild

    fromchild = property(fromchild)

    def outputsearch(self, p, timeout=None, linemode=False):
        """Search for 'p' in the command's output, while listening for more output from command, within 'timeout'

        'p' can be a list of re patterns or a single re pattern
           the value of a pattern can be an re string, or a compiled re object
        If 'timeout' is None, wait forever [*]

	'linemode' determines whether we search output line by line (as it comes), or all of the output in aggregate
        
        Return value:
        Did we match the output?
            Return a tuple (the pattern we matched, the string match)
        Otherwise (timeout/HUP) Return empty tuple ()

        Side effects:
        - If we HUP, we wait for the process to finish.
          You can check if the process is still running.

        - Output is collected and can be accessed by the output attribute [*]
        """

        patterns = []
        if not type(p) in (tuple, list):
            patterns.append(p)
        else:
            patterns += p

        # compile all patterns into re objects, but keep the original pattern object
        # so we can return it to the user when we match (friendlier interface)
        re_type = type(re.compile(""))
        for i in range(len(patterns)):
            if type(patterns[i]) is not re_type:
                patterns[i] = (re.compile(patterns[i]), patterns[i])
            else:
                patterns[i] = (patterns[i], patterns[i])

        def check_match():
            if linemode:
                while 1:
                    line = self._output.readline(True)
                    if not line:
                        return None

                    for pattern_re, pattern_orig in patterns:
                        match = pattern_re.search(line)
                        if match:
                            return pattern_orig, match

                    if not line.endswith('\n'):
                        return None

            else:
                # match against the entire buffered output
                for pattern_re, pattern_orig in patterns:
                    match = pattern_re.search(self._output.getvalue())
                    if match:
                        return pattern_orig, match

        # maybe we already match? (in buffered output)
        m = check_match()
        if m:
            return m

        ref = [()]
        started = time.time()

        def callback(self, buf):
            if buf:
                m = check_match()
                if m:
                    ref[0] = m
                    return False

            if buf == '':
                return False

            if timeout is not None:
                elapsed_time = time.time() - started
                if elapsed_time >= timeout:
                    return False

            return True

        fh = self.read(callback)
        return ref[0]

    def read(self, callback=None, callback_interval=0.1):
        """Read output from child.

        Args:
        'callback': callback(command, readbuf) every read loop or 
                    callback_interval (whichever comes sooner).

                    readbuf may be:
                     
                    1) a string
                    2) None (no input during callback_interval)
                    2) an empty string (EOF)

                    If callbacks returns False, stop reading

        Return read bytes.

        """

        if not callback:
            return self.fromchild.read()

        sio = StringIO()
        while True:

            output = self.fromchild.read(timeout=callback_interval)
            if output:
                sio.write(output)

            finished = callback(self, output) is False
            if finished:
                return sio.getvalue()

            if not self.running:
                break

            if output == '':
                self.wait()
                break

        if output != '':  # no leftovers if EOF
            leftovers = self.fromchild.read()
            sio.write(leftovers)
            callback(self, leftovers)

        return sio.getvalue()

    def __repr__(self):
        return "Command(%s)" % repr(self._cmd)

    def __str__(self):
        if isinstance(self._cmd, str):
            return self._cmd

        return fmt_argv(self._cmd)
Example #3
0
 def __init__(self):
     self.buffer = FIFOBuffer()
     self.parsed_header = False
     self.type = ''
     self.length = -1
Example #4
0
class Message(object):
    """
    Base class that for frontend or backend messages. Data is parsed 
    via the consume() method, and the message can be persisted for the wire
    using the serialize() method. 

    Instances of this class will have a variable set of properties, as 
    determined by the self.type field. Messages that are not yet completely 
    parsed will also have an inconsistent set of properties. 

    """
    def __init__(self):
        self.buffer = FIFOBuffer()
        self.parsed_header = False
        self.type = ''
        self.length = -1

    def consume(self, data):
        """
        Parses the provided data into this message instance. Returns a 2-tuple,
        of which the first element is a boolean indicating whether or not the 
        message was completely parsed. 

        If the message is not done parsing, the second element of the return 
        tuple is not meaningful. Otherwise, it contains the unconsumed data. 
        """
        self.buffer.append(data)

        if not self.parse_header():
            # not done.
            return False, ''

        if len(self.buffer) < self.length:
            # not done.
            return False, ''

        self.extra = self.buffer.truncate(self.length)
        self.parse_body()
        return True, self.extra

    def parse_body(self):
        """
        Provides a default parse function, which just places the amount of
        data as specified by the length field into the self.data property. 

        Derived classes can provide parse_* functions named for particular 
        type codes, which can do additional processing. 
        """
        def nothing():
            pass

        self.data = self.buffer.raw_value()[self.buffer.pos:self.length]
        getattr(self, 'parse_' + self.type, nothing)()

    def parse_header(self):
        """
        Parses the header out of the buffer. Returns true if successful, 
        false if there's not enough data yet. 
        """
        if self.parsed_header:
            return True

        if len(self.buffer.remainder()) < 5:
            # Not enough data for the minimum header yet.
            return False

        t = self.buffer.get_char()
        if ord(t) != 0:
            # this is an ordinary packet.
            self.type = t

            # add one, wire length doesn't include the type byte.
            self.length = self.buffer.get_int32() + 1
        else:
            if not self.parse_special_header():
                return False

        self.parsed_header = True
        return True

    def parse_special_header(self):
        """
        Parses irregular message types, meaning messages that do not start
        with a single-character identifier and a length. 
        """
        self.raise_unknown()

    def raise_unknown(self):
        raise ValueError(
            'Unknown %s packet: %r' %
            (self.__class__.__name__, self.buffer.raw_value()[:200]))

    def serialize(self):
        """
        Returns the data that should be written to the wire for this messag
        """
        return self.buffer.raw_value()[:self.length]

    def parseDict(self, data=None):
        """
        Parses a set of zero-delimited name, value pairs in data (or, by default,
        self.data) into a dict. 
        """
        data = data or self.data
        params = [x for x in data.split('\x00') if x]
        return dict([(k, v) for k, v in zip(params[::2], params[1::2])])

    def __str__(self):
        """
        Returns a human-readable form of the message. Dispatches to str_<type>
        functions, if they exist on the instance. 
        """
        return getattr(self, 'str_' + self.type, lambda: self.type)()
Example #5
0
 def __init__(self):
     self.buffer = FIFOBuffer()
     self.parsed_header = False
     self.type = ''
     self.length = -1
Example #6
0
class Message(object):
    """
    Base class that for frontend or backend messages. Data is parsed 
    via the consume() method, and the message can be persisted for the wire
    using the serialize() method. 

    Instances of this class will have a variable set of properties, as 
    determined by the self.type field. Messages that are not yet completely 
    parsed will also have an inconsistent set of properties. 

    """

    def __init__(self):
        self.buffer = FIFOBuffer()
        self.parsed_header = False
        self.type = ''
        self.length = -1

    
    def consume(self, data):
        """
        Parses the provided data into this message instance. Returns a 2-tuple,
        of which the first element is a boolean indicating whether or not the 
        message was completely parsed. 

        If the message is not done parsing, the second element of the return 
        tuple is not meaningful. Otherwise, it contains the unconsumed data. 
        """
        self.buffer.append(data)

        if not self.parse_header():
            # not done. 
            return False, ''

        if len(self.buffer) < self.length:
            # not done. 
            return False, ''
    
        self.extra = self.buffer.truncate(self.length)
        self.parse_body()
        return True, self.extra


    def parse_body(self):
        """
        Provides a default parse function, which just places the amount of
        data as specified by the length field into the self.data property. 

        Derived classes can provide parse_* functions named for particular 
        type codes, which can do additional processing. 
        """
        def nothing():
            pass

        self.data = self.buffer.raw_value()[self.buffer.pos:self.length]
        getattr(self, 'parse_' + self.type, nothing)()


    def parse_header(self):
        """
        Parses the header out of the buffer. Returns true if successful, 
        false if there's not enough data yet. 
        """
        if self.parsed_header:
            return True

        if len(self.buffer.remainder()) < 5:
            # Not enough data for the minimum header yet. 
            return False

        t = self.buffer.get_char()
        if ord(t) != 0:
            # this is an ordinary packet.
            self.type = t

            # add one, wire length doesn't include the type byte. 
            self.length = self.buffer.get_int32() + 1
        else:
            if not self.parse_special_header():
                return False

        self.parsed_header = True
        return True


    def parse_special_header(self):
        """
        Parses irregular message types, meaning messages that do not start
        with a single-character identifier and a length. 
        """
        self.raise_unknown()


    def raise_unknown(self):
        raise ValueError(
            'Unknown %s packet: %r' % (
                self.__class__.__name__, 
                self.buffer.raw_value()[:200]))


    def serialize(self):
        """
        Returns the data that should be written to the wire for this messag
        """
        return self.buffer.raw_value()[:self.length]


    def parseDict(self, data=None):
        """
        Parses a set of zero-delimited name, value pairs in data (or, by default,
        self.data) into a dict. 
        """
        data = data or self.data
        params = [x for x in data.split('\x00') if x]
        return dict([(k, v) for k, v in zip(params[::2], params[1::2])])



    def __str__(self):
        """
        Returns a human-readable form of the message. Dispatches to str_<type>
        functions, if they exist on the instance. 
        """
        return getattr(self, 'str_' + self.type, lambda: self.type)()