def __init__(self, cmd, runas=None, pty=False, setpgrp=None, debug=False):
"""Args:
'cmd' what command to execute
Can be a list ("/bin/ls", "-la")
or a string "/bin/ls" (will be passed to sh -c)
'pty' do we allocate a pty for command?
'runas' user we run as (set user, set groups, etc.)
'setpgrp' do we setpgrp in child? (create its own process group)
"""
self.ppid = os.getpid()
self._child = None
self._child = popen4.Popen4(cmd, 0, pty, runas, setpgrp)
self.tochild = self._child.tochild
self._fromchild = None
self.pid = self._child.pid
self._setpgrp = setpgrp
self._debug = debug
self._cmd = cmd
self._output = FIFOBuffer()
self._dprint("# command started (pid=%d, pty=%s): %s" %
(self._child.pid, repr(pty), cmd))
class Command(object):
"""Convenience module for executing a command
attribute notes::
'exitcode' - None if the process hasn't exited, exitcode otherwise
'output' - None if the process hasn't exited and fromchild hasn't
been accessed, the full output of the process
'running' - True if process is still running, False otherwise
'terminated' - Returns signal number if terminated, None otherwise
Usage example::
c = Command("./test.py")
if c.running:
c.wait()
assert c.running is False
print c.output
c = Command("./test.py")
# Unless you read from command.fromchild or use
# command.outputsearch() command.output will be None until the
# command finishes.
while c.output is None:
time.sleep(1)
print "output = '%s', exitcode = %d" % (c.output, c.exitcode)
c = Command("cat", pty=True)
print >> c.tochild, "test"
print c.fromchild.readline(),
"""
class Error(Exception):
pass
class _ChildObserver(Observer):
def __init__(self, outputbuf, debug=False):
self.debug = debug
self.outputbuf = outputbuf
def _dprint(self, event, msg):
if self.debug:
print("# EVENT '%s':\n%s" % (event, msg), file=sys.stderr)
def notify(self, subject, event, val):
if event in ('read', 'readline'):
self._dprint(event, val)
self.outputbuf.write(val)
elif event in ('readlines', 'xreadlines'):
self._dprint(event, "".join(val))
self.outputbuf.write("".join(val))
def __init__(self, cmd, runas=None, pty=False, setpgrp=None, debug=False):
"""Args:
'cmd' what command to execute
Can be a list ("/bin/ls", "-la")
or a string "/bin/ls" (will be passed to sh -c)
'pty' do we allocate a pty for command?
'runas' user we run as (set user, set groups, etc.)
'setpgrp' do we setpgrp in child? (create its own process group)
"""
self.ppid = os.getpid()
self._child = None
self._child = popen4.Popen4(cmd, 0, pty, runas, setpgrp)
self.tochild = self._child.tochild
self._fromchild = None
self.pid = self._child.pid
self._setpgrp = setpgrp
self._debug = debug
self._cmd = cmd
self._output = FIFOBuffer()
self._dprint("# command started (pid=%d, pty=%s): %s" %
(self._child.pid, repr(pty), cmd))
def __del__(self):
if not self._child:
return
# don't terminate() a process we didn't start
if os.getpid() == self.ppid:
self.terminate()
def _dprint(self, msg):
if self._debug:
print(msg, file=sys.stderr)
def terminate(self, gracetime=0, sig=signal.SIGTERM):
"""terminate command. kills command with 'sig', then sleeps for 'gracetime', before sending SIGKILL
"""
if self.running:
if self._child.pty:
cc_magic = termios.tcgetattr(self._child.tochild.fileno())[-1]
ctrl_c = cc_magic[termios.VINTR]
self._child.tochild.write(ctrl_c)
pid = self.pid
if self._setpgrp:
pid = -pid
try:
os.kill(pid, sig)
except OSError as e:
if e[0] != errno.EPERM or \
not self._child.pty or \
not self.wait(timeout=6, poll_interval=0.1):
raise
return
for i in range(gracetime):
if not self.running:
return
time.sleep(1)
if self.running:
os.kill(pid, signal.SIGKILL)
if not self.wait(timeout=3, poll_interval=0.1):
raise self.Error("process just won't die!")
self._dprint("# command (pid %d) terminated" % self._child.pid)
def terminated(self):
status = self._child.poll()
if not os.WIFSIGNALED(status):
return None
return os.WTERMSIG(status)
terminated = property(terminated)
def running(self):
if self._child.poll() == -1:
return True
return False
running = property(running)
def exitcode(self):
if self.running:
return None
status = self._child.poll()
if not os.WIFEXITED(status):
return None
return os.WEXITSTATUS(status)
exitcode = property(exitcode)
def wait(self, timeout=None, poll_interval=0.2, callback=None):
"""wait for process to finish executing.
'timeout' is how long we wait in seconds (None is forever)
'poll_interval' is how long we sleep between checks to see if process has finished
'callback': you can use callback to check for other conditions (e.g., besides timeout) and stop wait early.
return value: did the process finish? True/False
"""
if not self.running:
return True
if timeout is None:
self._child.wait()
return True
else:
start = time.time()
while time.time() - start < timeout:
if callback and callback() is False:
return False
if not self.running:
return True
time.sleep(poll_interval)
return False
def output(self):
if len(self._output):
return self._output.getvalue()
if self.running:
return None
# this will read into self._output via _ChildObserver
self.fromchild.read()
return self._output.getvalue()
output = property(output)
def fromchild(self):
"""return the command's filehandler.
NOTE: this file handler magically updates self._output"""
if self._fromchild:
return self._fromchild
fh = FileEventAdaptor(self._child.fromchild)
fh.addObserver(self._ChildObserver(self._output, self._debug))
fh = FileEnhancedRead(fh)
self._fromchild = fh
return self._fromchild
fromchild = property(fromchild)
def outputsearch(self, p, timeout=None, linemode=False):
"""Search for 'p' in the command's output, while listening for more output from command, within 'timeout'
'p' can be a list of re patterns or a single re pattern
the value of a pattern can be an re string, or a compiled re object
If 'timeout' is None, wait forever [*]
'linemode' determines whether we search output line by line (as it comes), or all of the output in aggregate
Return value:
Did we match the output?
Return a tuple (the pattern we matched, the string match)
Otherwise (timeout/HUP) Return empty tuple ()
Side effects:
- If we HUP, we wait for the process to finish.
You can check if the process is still running.
- Output is collected and can be accessed by the output attribute [*]
"""
patterns = []
if not type(p) in (tuple, list):
patterns.append(p)
else:
patterns += p
# compile all patterns into re objects, but keep the original pattern object
# so we can return it to the user when we match (friendlier interface)
re_type = type(re.compile(""))
for i in range(len(patterns)):
if type(patterns[i]) is not re_type:
patterns[i] = (re.compile(patterns[i]), patterns[i])
else:
patterns[i] = (patterns[i], patterns[i])
def check_match():
if linemode:
while 1:
line = self._output.readline(True)
if not line:
return None
for pattern_re, pattern_orig in patterns:
match = pattern_re.search(line)
if match:
return pattern_orig, match
if not line.endswith('\n'):
return None
else:
# match against the entire buffered output
for pattern_re, pattern_orig in patterns:
match = pattern_re.search(self._output.getvalue())
if match:
return pattern_orig, match
# maybe we already match? (in buffered output)
m = check_match()
if m:
return m
ref = [()]
started = time.time()
def callback(self, buf):
if buf:
m = check_match()
if m:
ref[0] = m
return False
if buf == '':
return False
if timeout is not None:
elapsed_time = time.time() - started
if elapsed_time >= timeout:
return False
return True
fh = self.read(callback)
return ref[0]
def read(self, callback=None, callback_interval=0.1):
"""Read output from child.
Args:
'callback': callback(command, readbuf) every read loop or
callback_interval (whichever comes sooner).
readbuf may be:
1) a string
2) None (no input during callback_interval)
2) an empty string (EOF)
If callbacks returns False, stop reading
Return read bytes.
"""
if not callback:
return self.fromchild.read()
sio = StringIO()
while True:
output = self.fromchild.read(timeout=callback_interval)
if output:
sio.write(output)
finished = callback(self, output) is False
if finished:
return sio.getvalue()
if not self.running:
break
if output == '':
self.wait()
break
if output != '': # no leftovers if EOF
leftovers = self.fromchild.read()
sio.write(leftovers)
callback(self, leftovers)
return sio.getvalue()
def __repr__(self):
return "Command(%s)" % repr(self._cmd)
def __str__(self):
if isinstance(self._cmd, str):
return self._cmd
return fmt_argv(self._cmd)
def __init__(self):
self.buffer = FIFOBuffer()
self.parsed_header = False
self.type = ''
self.length = -1
class Message(object):
"""
Base class that for frontend or backend messages. Data is parsed
via the consume() method, and the message can be persisted for the wire
using the serialize() method.
Instances of this class will have a variable set of properties, as
determined by the self.type field. Messages that are not yet completely
parsed will also have an inconsistent set of properties.
"""
def __init__(self):
self.buffer = FIFOBuffer()
self.parsed_header = False
self.type = ''
self.length = -1
def consume(self, data):
"""
Parses the provided data into this message instance. Returns a 2-tuple,
of which the first element is a boolean indicating whether or not the
message was completely parsed.
If the message is not done parsing, the second element of the return
tuple is not meaningful. Otherwise, it contains the unconsumed data.
"""
self.buffer.append(data)
if not self.parse_header():
# not done.
return False, ''
if len(self.buffer) < self.length:
# not done.
return False, ''
self.extra = self.buffer.truncate(self.length)
self.parse_body()
return True, self.extra
def parse_body(self):
"""
Provides a default parse function, which just places the amount of
data as specified by the length field into the self.data property.
Derived classes can provide parse_* functions named for particular
type codes, which can do additional processing.
"""
def nothing():
pass
self.data = self.buffer.raw_value()[self.buffer.pos:self.length]
getattr(self, 'parse_' + self.type, nothing)()
def parse_header(self):
"""
Parses the header out of the buffer. Returns true if successful,
false if there's not enough data yet.
"""
if self.parsed_header:
return True
if len(self.buffer.remainder()) < 5:
# Not enough data for the minimum header yet.
return False
t = self.buffer.get_char()
if ord(t) != 0:
# this is an ordinary packet.
self.type = t
# add one, wire length doesn't include the type byte.
self.length = self.buffer.get_int32() + 1
else:
if not self.parse_special_header():
return False
self.parsed_header = True
return True
def parse_special_header(self):
"""
Parses irregular message types, meaning messages that do not start
with a single-character identifier and a length.
"""
self.raise_unknown()
def raise_unknown(self):
raise ValueError(
'Unknown %s packet: %r' %
(self.__class__.__name__, self.buffer.raw_value()[:200]))
def serialize(self):
"""
Returns the data that should be written to the wire for this messag
"""
return self.buffer.raw_value()[:self.length]
def parseDict(self, data=None):
"""
Parses a set of zero-delimited name, value pairs in data (or, by default,
self.data) into a dict.
"""
data = data or self.data
params = [x for x in data.split('\x00') if x]
return dict([(k, v) for k, v in zip(params[::2], params[1::2])])
def __str__(self):
"""
Returns a human-readable form of the message. Dispatches to str_<type>
functions, if they exist on the instance.
"""
return getattr(self, 'str_' + self.type, lambda: self.type)()
def __init__(self):
self.buffer = FIFOBuffer()
self.parsed_header = False
self.type = ''
self.length = -1
class Message(object):
"""
Base class that for frontend or backend messages. Data is parsed
via the consume() method, and the message can be persisted for the wire
using the serialize() method.
Instances of this class will have a variable set of properties, as
determined by the self.type field. Messages that are not yet completely
parsed will also have an inconsistent set of properties.
"""
def __init__(self):
self.buffer = FIFOBuffer()
self.parsed_header = False
self.type = ''
self.length = -1
def consume(self, data):
"""
Parses the provided data into this message instance. Returns a 2-tuple,
of which the first element is a boolean indicating whether or not the
message was completely parsed.
If the message is not done parsing, the second element of the return
tuple is not meaningful. Otherwise, it contains the unconsumed data.
"""
self.buffer.append(data)
if not self.parse_header():
# not done.
return False, ''
if len(self.buffer) < self.length:
# not done.
return False, ''
self.extra = self.buffer.truncate(self.length)
self.parse_body()
return True, self.extra
def parse_body(self):
"""
Provides a default parse function, which just places the amount of
data as specified by the length field into the self.data property.
Derived classes can provide parse_* functions named for particular
type codes, which can do additional processing.
"""
def nothing():
pass
self.data = self.buffer.raw_value()[self.buffer.pos:self.length]
getattr(self, 'parse_' + self.type, nothing)()
def parse_header(self):
"""
Parses the header out of the buffer. Returns true if successful,
false if there's not enough data yet.
"""
if self.parsed_header:
return True
if len(self.buffer.remainder()) < 5:
# Not enough data for the minimum header yet.
return False
t = self.buffer.get_char()
if ord(t) != 0:
# this is an ordinary packet.
self.type = t
# add one, wire length doesn't include the type byte.
self.length = self.buffer.get_int32() + 1
else:
if not self.parse_special_header():
return False
self.parsed_header = True
return True
def parse_special_header(self):
"""
Parses irregular message types, meaning messages that do not start
with a single-character identifier and a length.
"""
self.raise_unknown()
def raise_unknown(self):
raise ValueError(
'Unknown %s packet: %r' % (
self.__class__.__name__,
self.buffer.raw_value()[:200]))
def serialize(self):
"""
Returns the data that should be written to the wire for this messag
"""
return self.buffer.raw_value()[:self.length]
def parseDict(self, data=None):
"""
Parses a set of zero-delimited name, value pairs in data (or, by default,
self.data) into a dict.
"""
data = data or self.data
params = [x for x in data.split('\x00') if x]
return dict([(k, v) for k, v in zip(params[::2], params[1::2])])
def __str__(self):
"""
Returns a human-readable form of the message. Dispatches to str_<type>
functions, if they exist on the instance.
"""
return getattr(self, 'str_' + self.type, lambda: self.type)()