def __open_process(self, logger, cmd, env, cwd, **kwargs):
if logger:
logger.info('Running command: %r' % cmd)
if env:
logger.info('With env (showing only all caps):')
for key, val in env.iteritems():
if re.match(r'[A-Z0-9_]+', key):
logger.info(' %-20s = %s' % (key,val))
logger.info('With CWD: %s' % (cwd or os.getcwd()))
logger.info('-' * 100)
# provide tty to enable line buffering.
master_out_fd, slave_out_fd = pty.openpty()
master_err_fd, slave_err_fd = pty.openpty()
process = subprocess.Popen(
cmd,
cwd=cwd,
env=env,
shell=False,
bufsize=1,
stderr=slave_err_fd,
stdout=slave_out_fd,
close_fds=True)
return (process, (master_out_fd, slave_out_fd), (master_err_fd, slave_err_fd))
def mkpty():
master1, slave = pty.openpty()
slaveName1 = os.ttyname(slave)
master2, slave = pty.openpty()
slaveName2 = os.ttyname(slave)
print '\nslave device names: ', slaveName1, slaveName2
return master1, master2
def __init__(self, ip=get_ipython()):
# For the registration
self.shell = ip
self.nbOutStream = sys.stdout
self.nbErrStream = sys.stderr
self.pyOutStream = sys.__stdout__
self.pyErrStream = sys.__stderr__
self.outStreamPipe_in = pty.openpty()[1]
self.errStreamPipe_in = pty.openpty()[1]
os.dup2(self.outStreamPipe_in, self.pyOutStream.fileno())
os.dup2(self.errStreamPipe_in, self.pyErrStream.fileno())
self.ioHandler = handlers.IOHandler()
self.flag = True
self.outString = ""
self.errString = ""
self.asyncCapturer = handlers.Runner(self.syncCapture)
self.isFirstPreExecute = True
self.isFirstPostExecute = True
def run(self):
try:
# use a pty. This enforces unbuffered output and thus
# allows for fast update
master_out_fd, slave_out_fd = pty.openpty()
master_in_fd, self.slave_in_fd = pty.openpty()
self.proc = subprocess.Popen(self.cmd, stdin=master_in_fd, stdout=slave_out_fd, stderr=slave_out_fd)
except:
self.finished.emit(False)
return
# listen to process' output
while self.proc.poll() == None:
try:
if select.select([master_out_fd], [], [master_out_fd], .1)[0]:
output = os.read(master_out_fd, 100)
if output: self.output(str(output, "utf-8"))
except InterruptedError:
pass
os.close(master_out_fd)
os.close(slave_out_fd)
os.close(master_in_fd)
os.close(self.slave_in_fd)
self.finished.emit(self.proc.wait() == 0)
def __init__(self, cmd, capturestderr=False, bufsize=-1):
""" Popen3 class (isn't this actually Popen4, capturestderr = False?) that uses ptys
instead of pipes, to allow inline reading (instead of potential i/o buffering) of
output from the child process. It also stores the cmd its running (as a string)
and the thread that created the object, for later use """
import pty
# NOTE: most of this is cutnpaste from Popen3 minus the openpty calls
# popen2._cleanup()
self.prettyCmd = cmd
cmd = self.parseCmdToList(cmd)
self.cmd = cmd
self.threadIdent = thread.get_ident()
p2cread, p2cwrite = pty.openpty()
c2pread, c2pwrite = pty.openpty()
if capturestderr:
errout, errin = pty.openpty()
self.pid = os.fork()
if self.pid == 0:
# Child
os.dup2(p2cread, 0)
os.dup2(c2pwrite, 1)
if capturestderr:
os.dup2(errin, 2)
self._run_child(cmd)
os.close(p2cread)
self.tochild = os.fdopen(p2cwrite, "w", bufsize)
os.close(c2pwrite)
self.fromchild = os.fdopen(c2pread, "r", bufsize)
if capturestderr:
os.close(errin)
self.childerr = os.fdopen(errout, "r", bufsize)
else:
self.childerr = None
def __init__(self, args, cwd=None, env=None, stdin=False,
echo_stdout=True, echo_stderr=True,
capture_stdout=False, capture_stderr=False):
if cwd is not None:
cwd = str(cwd)
logging.debug('Running {}...'.format(args[0]))
logging.debug('Parameters: {}'.format(args))
logging.debug('Working directory: {}'.format(cwd))
logging.debug('Environment: {}'.format(env))
logging.debug('Echo: stdout: {}, stderr: {}'.format(echo_stdout, echo_stderr))
self.args = args
self.buffer_stdout = bytearray()
self.buffer_stderr = bytearray()
master_stdout, slave_stdout = pty.openpty()
master_stderr, slave_stderr = pty.openpty()
Process.set_nonblocking(master_stdout)
Process.set_nonblocking(master_stderr)
self.process = subprocess.Popen(args, bufsize=0, cwd=cwd, env=env,
stdin=stdin, stdout=slave_stdout, stderr=slave_stderr)
pass_to_stdout = sys.stdout.buffer if echo_stdout else None
pass_to_stderr = sys.stderr.buffer if echo_stderr else None
self._reader_stdout = self.reader(master_stdout, capture_stdout,
self.buffer_stdout, pass_to_stdout)
self._reader_stderr = self.reader(master_stderr, capture_stderr,
self.buffer_stderr, pass_to_stderr)
def tty_capture(cmd, bytes_input):
"""Capture the output of cmd with bytes_input to stdin,
with stdin, stdout and stderr as TTYs."""
mo, so = pty.openpty() # provide tty to enable line-buffering
me, se = pty.openpty()
mi, si = pty.openpty()
fdmap = {mo: 'stdout', me: 'stderr', mi: 'stdin'}
p = subprocess.Popen(
cmd, bufsize=1, stdin=si, stdout=so, stderr=se, close_fds=True)
os.write(mi, bytes_input)
timeout = .04 # seconds
res = {'stdout': b'', 'stderr': b''}
while True:
ready, _, _ = select.select([mo, me], [], [], timeout)
if ready:
for fd in ready:
data = os.read(fd, 512)
if not data:
break
res[fdmap[fd]] += data
elif p.poll() is not None: # select timed-out
break # p exited
for fd in [si, so, se, mi, mo, me]:
os.close(fd) # can't do it sooner: it leads to errno.EIO error
p.wait()
return p.returncode, res['stdout'], res['stderr']
def _create_pty_or_pipe(copy_term_size=None):
"""
Try to create a pty and if then fails then create a normal
pipe instead.
@param copy_term_size: If a tty file descriptor is given
then the term size will be copied to the pty.
@type copy_term_size: int
@rtype: tuple
@returns: A tuple of (is_pty, master_fd, slave_fd) where
is_pty is True if a pty was successfully allocated, and
False if a normal pipe was allocated.
"""
got_pty = False
global _disable_openpty, _fbsd_test_pty
if _fbsd_test_pty and not _disable_openpty:
# Test for python openpty breakage after freebsd7 to freebsd8
# upgrade, which results in a 'Function not implemented' error
# and the process being killed.
pid = os.fork()
if pid == 0:
pty.openpty()
os._exit(os.EX_OK)
pid, status = os.waitpid(pid, 0)
if (status & 0xff) == 140:
_disable_openpty = True
_fbsd_test_pty = False
if _disable_openpty:
master_fd, slave_fd = os.pipe()
else:
try:
master_fd, slave_fd = pty.openpty()
got_pty = True
except EnvironmentError as e:
_disable_openpty = True
writemsg("openpty failed: '%s'\n" % str(e),
noiselevel=-1)
del e
master_fd, slave_fd = os.pipe()
if got_pty:
# Disable post-processing of output since otherwise weird
# things like \n -> \r\n transformations may occur.
mode = termios.tcgetattr(slave_fd)
mode[1] &= ~termios.OPOST
termios.tcsetattr(slave_fd, termios.TCSANOW, mode)
if got_pty and \
copy_term_size is not None and \
os.isatty(copy_term_size):
rows, columns = get_term_size()
set_term_size(rows, columns, slave_fd)
return (got_pty, master_fd, slave_fd)
def _execute_process_pty(cmd, cwd, env, shell, stderr_to_stdout=True):
stdout_master, stdout_slave = None, None
stderr_master, stderr_slave = None, None
fds_to_close = [stdout_master, stdout_slave, stderr_master, stderr_slave]
try:
stdout_master, stdout_slave = pty.openpty()
if stderr_to_stdout:
stderr_master, stderr_slave = stdout_master, stdout_slave
else:
stderr_master, stderr_slave = pty.openpty()
p = None
while p is None:
try:
p = Popen(
cmd,
stdin=stdout_slave, stdout=stderr_slave, stderr=STDOUT,
cwd=cwd, env=env, shell=shell, close_fds=False)
except OSError as exc:
# This can happen if a file you are trying to execute is being
# written to simultaneously on Linux
# (doesn't appear to happen on OS X)
# It seems like the best strategy is to just try again later
# Worst case is that the file eventually gets deleted, then a
# different OSError would occur.
if 'Text file busy' in '{0}'.format(exc):
# This is a transient error, try again shortly
time.sleep(0.01)
continue
raise
# This causes the below select to exit when the subprocess closes.
# On Linux, this sometimes causes Errno 5 OSError's when os.read
# is called from within _yield_data, so on Linux _yield_data
# catches and passes on that particular OSError.
os.close(stdout_slave)
if not stderr_to_stdout:
os.close(stderr_slave)
left_overs = {stdout_master: b'', stderr_master: b''}
fds = [stdout_master]
if stderr_master != stdout_master:
fds.append(stderr_master)
finally:
# Make sure we don't leak file descriptors
_close_fds(fds_to_close)
# The linesep with pty's always seems to be "\r\n", even on OS X
return _yield_data(p, fds, left_overs, "\r\n", fds_to_close)
def __init__(self, path, args, stdout_cb, stderr_cb, exited_cb, timeout_cb, timeout=15):
threading.Thread.__init__(self)
self.path = os.path.abspath(path)
if HAS_SANDBOX_SUPPORT:
self.sandboxpy = os.path.join(os.path.dirname(os.path.abspath(__file__)), "MiniSandbox.py")
self.arguments = ["python", self.sandboxpy, os.path.dirname(self.path), self.path]
else:
self.sandboxpy = os.path.join(os.path.dirname(os.path.abspath(__file__)), "NoSandbox.py")
self.arguments = ["python", self.sandboxpy, os.path.dirname(self.path), self.path]
self.arguments.extend(args)
self.master, slave = pty.openpty()
self.proc = subprocess.Popen(self.arguments, stdin=slave, stdout=slave, stderr=subprocess.PIPE, close_fds=True)
self.stdout_cb = stdout_cb
self.stderr_cb = stderr_cb
self.exited_cb = exited_cb
self.timeout_cb = timeout_cb
self.timeout = timeout
self.input_queue = []
def start_subprocess(self):
"""Start octave using a subprocess (no tty support)"""
errmsg = ('\n\nPlease install GNU Octave and put it in your path\n')
ON_POSIX = 'posix' in sys.builtin_module_names
if self.use_pty:
master, slave = pty.openpty()
self.wfid, self.rfid = master, master
rpipe, wpipe = slave, slave
else:
self.rfid, wpipe = os.pipe()
rpipe, self.wfid = os.pipe()
kwargs = dict(close_fds=ON_POSIX, bufsize=0, stdin=rpipe,
stderr=wpipe, stdout=wpipe)
if os.name == 'nt':
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
kwargs['startupinfo'] = startupinfo
try:
proc = subprocess.Popen(['octave', '-q', '--braindead'],
**kwargs)
except OSError: # pragma: no cover
raise Oct2PyError(errmsg)
else:
self.reader = _Reader(self.rfid, self.read_queue)
return proc
def handle(self):
masterFd, slaveFd = pty.openpty()
try:
# if we're not in the main thread, this will not work.
signal.signal(signal.SIGTTOU, signal.SIG_IGN)
except:
pass
pid = os.fork()
if pid:
os.close(masterFd)
raise SocketConnected(slaveFd, pid)
# make parent process the pty slave - the opposite of
# pty.fork(). In this setup, the parent process continues
# to act normally, while the child process performs the
# logging. This makes it simple to kill the logging process
# when we are done with it and restore the parent process to
# normal, unlogged operation.
else:
os.close(slaveFd)
try:
protocol = TelnetServerProtocolHandler(self.request, masterFd)
protocol.handle()
finally:
os.close(masterFd)
os._exit(1)
def __t_pty_tracker(self, trackerclass, **kwargs):
def __drain(masterf):
while True:
termdata = masterf.read(1024)
if len(termdata) == 0:
break
#
# - Allocate a pty
# - Create a thread to drain off the master side; without
# this, the slave side will block when trying to write.
# - Connect the prog tracker to the slave side
# - Set it running
#
(master, slave) = pty.openpty()
slavef = os.fdopen(slave, "w")
masterf = os.fdopen(master, "r")
t = threading.Thread(target=__drain, args=(masterf,))
t.start()
p = trackerclass(output_file=slavef, **kwargs)
progress.test_progress_tracker(p, gofast=True)
slavef.close()
t.join()
masterf.close()
def _has_sudo(self, result):
_master, slave = pty.openpty()
os.setsid()
fcntl.ioctl(slave, termios.TIOCSCTTY, 0)
out, err, exit = run_command(['sudo', '-l', '-U', self.user[USER_NAME],
'sudo'])
if exit == 0:
debug("User %s is allowed to run sudo" % self.user[USER_NAME])
# sudo allows a wide range of configurations, such as controlling
# which binaries the user can execute with sudo.
# For now, we will just check whether the user is allowed to run
# any command with sudo.
out, err, exit = run_command(['sudo', '-l', '-U',
self.user[USER_NAME]])
for line in out.split('\n'):
if line and re.search("(ALL)", line):
result.value = 1
debug("User %s can run any command with sudo" %
result.value)
return
debug("User %s can only run some commands with sudo" %
self.user[USER_NAME])
else:
debug("User %s is not allowed to run sudo" % self.user[USER_NAME])
def run(self):
self.ioloop = IOLoop.instance()
(master_fd, slave_fd) = pty.openpty()
# make stdout, stderr non-blocking
fcntl.fcntl(master_fd, fcntl.F_SETFL,
fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
self.master_fd = master_fd
self.master = os.fdopen(master_fd)
# listen to stdout, stderr
self.ioloop.add_handler(master_fd, self._handle_subprocess_stdout,
self.ioloop.READ)
slave = os.fdopen(slave_fd)
self.kwargs["stdout"] = slave
self.kwargs["stderr"] = slave
self.kwargs["close_fds"] = True
self.pipe = subprocess.Popen(*self.args, **self.kwargs)
self.stdin = self.pipe.stdin
# check for process exit
self.wait_callback = PeriodicCallback(self._wait_for_end, 250)
self.wait_callback.start()
def ssh_output(cmd):
""" Runs an SSH command and returns the stdout/stderr.
:param cmd: command to run
:type cmd: str
:rtype: (str, str)
"""
# ssh must run with stdin attached to a tty
master, slave = pty.openpty()
proc = subprocess.Popen(cmd,
stdin=slave,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=os.setsid,
close_fds=True,
shell=True)
os.close(slave)
# wait for the ssh connection
time.sleep(8)
# kill the whole process group
os.killpg(os.getpgid(proc.pid), 15)
os.close(master)
stdout, stderr = proc.communicate()
print('SSH STDOUT: {}'.format(stdout.decode('utf-8')))
print('SSH STDERR: {}'.format(stderr.decode('utf-8')))
return stdout, stderr
def run_command(cmd, cwd=None):
master, slave = pty.openpty()
p = None
while p is None:
try:
p = Popen(cmd, stdin=slave, stdout=slave, stderr=STDOUT, cwd=cwd)
except OSError as exc:
if "Text file busy" in str(exc):
# This is a transient error, try again shortly
time.sleep(0.01)
continue
raise
if sys.platform.startswith("darwin"):
os.close(slave) # This causes the below select to exit when the subprocess closes
left_over = b""
# Read data until the process is finished
while p.poll() is None:
incomming = left_over
rlist, wlist, xlist = select.select([master], [], [], 0.1)
if rlist:
incomming += os.read(master, 1024)
lines = incomming.splitlines(True) # keepends=True
data, left_over = process_incomming_lines(lines, left_over)
if data is None:
continue
yield data
# Done
os.close(master)
yield p.returncode
def __init__(self, cmd, logger):
self.thread = threading.Thread(target=TestProcess._run, args=(weakref.proxy(self),))
self.pty_master, self.pty_slave = pty.openpty()
self.logger = logger
self.process = None
self.traces = ""
self.cmd = cmd
def call_and_peek_output(cmd, shell=False):
import pty
import subprocess
master, slave = pty.openpty()
print cmd
p = subprocess.Popen(cmd, shell=shell, stdin=None, stdout=slave, close_fds=True)
os.close(slave)
line = ""
while True:
try:
ch = os.read(master, 1)
except OSError:
# We get this exception when the spawn process closes all references to the
# pty descriptor which we passed him to use for stdout
# (typically when it and its childs exit)
break
line += ch
if ch == '\n':
yield line
line = ""
if line:
yield line
ret = p.wait()
if ret:
raise subprocess.CalledProcessError(ret, cmd)
def __enter__(self):
# prepare standard file descriptors for raw manipulation
self.was_blocking = os.get_blocking(0)
os.set_blocking(0, False)
try:
self.terminal_attr_stdin = termios.tcgetattr(0)
self.terminal_attr_stdout = termios.tcgetattr(1)
self.terminal_attr_stderr = termios.tcgetattr(2)
tty.setraw(0)
tty.setraw(1)
tty.setraw(2)
except termios.error: # probably redirected
self.terminal_attr_stdin = None
# redirect standard file descriptors to new PTY
master, slave = pty.openpty()
os.set_blocking(master, False)
self.real_stdin = os.dup(0)
self.real_stdout = os.dup(1)
self.real_stderr = os.dup(2)
os.close(0)
os.close(1)
os.close(2)
os.dup2(slave, 0)
os.dup2(slave, 1)
os.dup2(slave, 2)
os.close(slave)
self.terminal_pipe = master
# start REPL in separate thread
threading.Thread(target=repl, args=(self,), daemon=True).start()
return self
def _get_pty_pair(self, encoding='ascii'):
master_fd, slave_fd = pty.openpty()
master = os.fdopen(master_fd, 'rb', 0)
out = os.fdopen(slave_fd, 'wb', 0)
master = io.TextIOWrapper(master)
out = io.TextIOWrapper(out)
return master, out
def test_signal_failure(monkeypatch):
import os
import pty
import signal
from pyrepl.unix_console import UnixConsole
def failing_signal(a, b):
raise ValueError
def really_failing_signal(a, b):
raise AssertionError
mfd, sfd = pty.openpty()
try:
with sane_term():
c = UnixConsole(sfd, sfd)
c.prepare()
c.restore()
monkeypatch.setattr(signal, 'signal', failing_signal)
c.prepare()
monkeypatch.setattr(signal, 'signal', really_failing_signal)
c.restore()
finally:
os.close(mfd)
os.close(sfd)
def _node_ssh_output(args):
# ssh must run with stdin attached to a tty
master, slave = pty.openpty()
cmd = ('ssh-agent /bin/bash -c ' +
'"ssh-add /host-home/.vagrant.d/insecure_private_key ' +
'2> /dev/null && dcos node ssh --option StrictHostKeyChecking=no' +
' {}"').format(' '.join(args))
proc = subprocess.Popen(cmd,
stdin=slave,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=os.setsid,
close_fds=True,
shell=True)
os.close(slave)
# wait for the ssh connection
time.sleep(8)
# kill the whole process group
os.killpg(os.getpgid(proc.pid), 15)
os.close(master)
return proc.communicate()
def lock(self):
"""Use conserver to lock the machine."""
# find out current status of console
debug.verbose('executing "console -i %s" to check state' %
self.get_machine_name())
proc = subprocess.Popen(["console", "-i", self.get_machine_name()],
stdout=subprocess.PIPE)
line = proc.communicate()[0]
assert(proc.returncode == 0)
# check that nobody else has it open for writing
myuser = getpass.getuser()
parts = line.strip().split(':')
conname, child, contype, details, users, state = parts[:6]
if users:
for userinfo in users.split(','):
mode, username, host, port = userinfo.split('@')[:4]
if 'w' in mode and username != myuser:
raise MachineLockedError # Machine is not free
# run a console in the background to 'hold' the lock and read output
debug.verbose('starting "console %s"' % self.get_machine_name())
# run on a PTY to work around terminal mangling code in console
(self.masterfd, slavefd) = pty.openpty()
self.lockprocess = subprocess.Popen(["console", self.get_machine_name()],
close_fds=True,
stdout=slavefd, stdin=slavefd)
os.close(slavefd)
# XXX: open in binary mode with no buffering
# otherwise select.select() may block when there is data in the buffer
self.console_out = os.fdopen(self.masterfd, 'rb', 0)
def spawn(self, argv=None, term=None):
if argv is None:
if "SHELL" in os.environ:
argv = [os.environ["SHELL"]]
elif "PATH" in os.environ: # searching sh in the path. It can be unusual like /system/bin/sh on android
for shell in ["bash", "sh", "ksh", "zsh", "csh", "ash"]:
for path in os.environ["PATH"].split(":"):
fullpath = os.path.join(path.strip(), shell)
if os.path.isfile(fullpath):
argv = [fullpath]
break
if argv:
break
if not argv:
argv = ["/bin/sh"]
if term is not None:
os.environ["TERM"] = term
master, slave = pty.openpty()
self.slave = slave
self.master = os.fdopen(master, "rb+wb", 0) # open file in an unbuffered mode
flags = fcntl.fcntl(self.master, fcntl.F_GETFL)
assert flags >= 0
flags = fcntl.fcntl(self.master, fcntl.F_SETFL, flags | os.O_NONBLOCK)
assert flags >= 0
self.prog = subprocess.Popen(
shell=False, args=argv, stdin=slave, stdout=slave, stderr=subprocess.STDOUT, preexec_fn=prepare
)
def __init__(self, parent=None):
"""Initialise Linux pseudo terminal
"""
QThread.__init__(self, parent)
self.master, self.slave = pty.openpty()
self.ptyname = os.ttyname(self.slave)
self.stop = False
def start_proc(self):
preexec = noop
stdin = stdout = stderr = None
cqlshlog.info("Spawning %r subprocess with args: %r and env: %r"
% (self.exe_path, self.args, self.env))
if self.realtty:
masterfd, slavefd = pty.openpty()
preexec = (lambda: set_controlling_pty(masterfd, slavefd))
self.proc = subprocess.Popen((self.exe_path,) + tuple(self.args),
env=self.env, preexec_fn=preexec,
stdin=stdin, stdout=stdout, stderr=stderr,
close_fds=False)
os.close(slavefd)
self.childpty = masterfd
self.send = self.send_tty
self.read = self.read_tty
else:
stdin = stdout = subprocess.PIPE
stderr = subprocess.STDOUT
self.proc = subprocess.Popen((self.exe_path,) + tuple(self.args),
env=self.env, stdin=stdin, stdout=stdout,
stderr=stderr, bufsize=0, close_fds=False)
self.send = self.send_pipe
if self.tty:
self.winpty = WinPty(self.proc.stdout)
self.read = self.read_winpty
else:
self.read = self.read_pipe
def command(cmd, exception=PyscaleError, sudo=False, shell=False): # fix unicode stuff cmd = str(cmd) # parse args if sudo: # XXX: --session-command vs --command(-c) # session-command seems to be better but is only available on CentOS & Co. # cmd = "su -c '%s'" % cmd cmd = "sudo -n bash -c '%s'" % cmd if not shell: cmd = shlex.split(cmd) # execute slave = None if sudo: # give su a pty master, slave = pty.openpty() out, err = GPopen(cmd, stdin=slave, stdout=sbp.PIPE, stderr = sbp.PIPE, shell=shell).communicate() # handle errors if not out and err: if exception: raise exception(err) else: print err return out
def __init_streams(self, stdin, stdout, stderr, unbuffered):
self.stdin = self.stdout = self.stderr = None
if unbuffered:
master, slave = pty.openpty()
if stdin is PIPE:
self._child_stdin, self._stdin = (slave, master) if unbuffered else os.pipe()
self.stdin = os.fdopen(self._stdin, 'w')
elif isinstance(stdin, int):
self._child_stdin, self._stdin = stdin, -1
elif stdin is not None:
self._child_stdin, self._stdin = stdin.fileno(), -1
else:
self._child_stdin = self._stdin = -1
if stdout is PIPE:
self._stdout, self._child_stdout = (master, slave) if unbuffered else os.pipe()
self.stdout = os.fdopen(self._stdout, 'r')
elif isinstance(stdout, int):
self._stdout, self._child_stdout = -1, stdout
elif stdout is not None:
self._stdout, self._child_stdout = -1, stdout.fileno()
else:
self._stdout = self._child_stdout = -1
if stderr is PIPE:
self._stderr, self._child_stderr = os.pipe()
self.stderr = os.fdopen(self._stderr, 'r')
elif isinstance(stderr, int):
self._stderr, self._child_stderr = -1, stderr
elif stderr is not None:
self._stderr, self._child_stderr = -1, stderr.fileno()
else:
self._stderr = self._child_stderr = -1
def start(self, conf_name):
self.conf = get_config(conf_name)
filename = export_conf(self.conf)
# Prepare server
## Prepare storage file
f = open(os.path.join(data_folder,'storage.cfg'), 'w')
# USELESS
#f.write('add_path .')
#f.write('add_path $USERDIR')
#f.write('add_path $DATADIR')
# END USELESS
f.write('add_path $CURRENTDIR')
f.close()
# Server Command
if self.conf['conf'].get('server', ''):
server_bin = self.conf['conf']['server']
else:
server_bin = "`which teeworlds-server`"
self.command = 'cd %s && %s -f %s' % (data_folder, server_bin, filename)
# Open pty
master, slave = pty.openpty()
# Launch server
self.process = Popen(self.command, shell=True, stdin=PIPE, stdout=slave, stderr=slave, close_fds=True)
# Init thread
self.server = TeeWorldsServer(self, master)
self.live_stats = LiveStats()
# Start Server
self.server.start()
# Start live stats thread
self.live_stats.start()
import json
import logging
import os
import pty
import re
import shlex
import subprocess
import tempfile
from fcntl import fcntl, F_GETFL, F_SETFL
from tests.cook import util
logger = logging.getLogger(__name__)
# Manually create a TTY that we can use as the default STDIN
_STDIN_TTY = pty.openpty()[1]
def decode(b):
"""Decodes as UTF-8"""
return b.decode('UTF-8')
def encode(o):
"""Encodes with UTF-8"""
return str(o).encode('UTF-8')
def stdout(cp):
"""Returns the UTF-8 decoded and stripped stdout of the given CompletedProcess"""
return decode(cp.stdout).strip()
def _make_popen_named_args(self, others=None):
if others is None:
(_, slave) = pty.openpty()
others = {'stdin': slave, 'stdout': slave, 'stderr': slave}
super(RunPty, self)._make_popen_named_args(others=others)
def __fork_ptys(self):
'''
Fork the PTY
The major difference from the python source is that we separate the
stdout from stderr output.
'''
stdout_parent_fd, stdout_child_fd = pty.openpty()
if stdout_parent_fd < 0 or stdout_child_fd < 0:
raise TerminalException('Failed to open a TTY for stdout')
stderr_parent_fd, stderr_child_fd = pty.openpty()
if stderr_parent_fd < 0 or stderr_child_fd < 0:
raise TerminalException('Failed to open a TTY for stderr')
pid = os.fork()
if pid < pty.CHILD:
raise TerminalException('Failed to fork')
elif pid == pty.CHILD:
# Child.
# Close parent FDs
os.close(stdout_parent_fd)
os.close(stderr_parent_fd)
salt.utils.reinit_crypto()
# ----- Make STDOUT the controlling PTY --------------------->
child_name = os.ttyname(stdout_child_fd)
# Disconnect from controlling tty. Harmless if not already
# connected
try:
tty_fd = os.open('/dev/tty', os.O_RDWR | os.O_NOCTTY)
if tty_fd >= 0:
os.close(tty_fd)
# which exception, shouldn't we catch explicitly .. ?
except: # pylint: disable=W0702
# Already disconnected. This happens if running inside cron
pass
# New session!
os.setsid()
# Verify we are disconnected from controlling tty
# by attempting to open it again.
try:
tty_fd = os.open('/dev/tty', os.O_RDWR | os.O_NOCTTY)
if tty_fd >= 0:
os.close(tty_fd)
raise TerminalException(
'Failed to disconnect from controlling tty. It is '
'still possible to open /dev/tty.')
# which exception, shouldn't we catch explicitly .. ?
except: # pylint: disable=W0702
# Good! We are disconnected from a controlling tty.
pass
# Verify we can open child pty.
tty_fd = os.open(child_name, os.O_RDWR)
if tty_fd < 0:
raise TerminalException(
'Could not open child pty, {0}'.format(child_name))
else:
os.close(tty_fd)
# Verify we now have a controlling tty.
if os.name != 'posix':
# Only do this check in not BSD-like operating systems. BSD-like operating systems breaks at this point
tty_fd = os.open('/dev/tty', os.O_WRONLY)
if tty_fd < 0:
raise TerminalException(
'Could not open controlling tty, /dev/tty')
else:
os.close(tty_fd)
# <---- Make STDOUT the controlling PTY ----------------------
# ----- Duplicate Descriptors ------------------------------->
os.dup2(stdout_child_fd, pty.STDIN_FILENO)
os.dup2(stdout_child_fd, pty.STDOUT_FILENO)
os.dup2(stderr_child_fd, pty.STDERR_FILENO)
# <---- Duplicate Descriptors --------------------------------
else:
# Parent. Close Child PTY's
salt.utils.reinit_crypto()
os.close(stdout_child_fd)
os.close(stderr_child_fd)
return pid, stdout_parent_fd, stderr_parent_fd
def inpty(ctx: Context,
cmd: Union[str, List],
env: Dict = None,
log: ByteString = None):
""" inpty polls colorful stdout of poetry to the terminal
This function executes command and use select + pty to pull process stdout and stderr to file-object 'log' in non-blocking way, so it can simultaneously logging to stdout and close by itself after the process is done.
Args:
ctx (click.Context): a click Context object which is used to control the flow
cmd (str, List): command that will be passed to a subprocess
env (Dict): copy current env settings to pty pseudo one to ensure process
log (ByteString): A disposable container to collect stdout which is polled from the subprocess
Returns:
returncode (int): Result status of the process
Raise:
click.Abort: Abort by click if encounter error(s)
"""
try:
exec_env = {}
exec_env.update(os.environ)
# copy the current OS environment into the local environment
if env is not None:
exec_env.update(env)
# create master & slave pipes to receive stdout and stderr from process
master, slave = pty.openpty()
# check cmd is a list, otherwise if try to split a string and check it can be well reformat
if not isinstance(cmd, list) and isinstance(cmd, str):
# First check it will the same, then keep processing, otherwise abort the process
assert cmd == " ".join(split(cmd)), ctx.abort()
cmd = split(cmd)
p = Popen(cmd, shell=False, env=exec_env, stdout=slave, stderr=slave)
# Loop while the process is executing
while p.poll() is None:
# Loop long as the selct mechanism indicates there is data to be read from the buffer
while len(select([master], [], [], 0)[0]) == 1:
# Read up to a 1 KB chunk of data
buf = os.read(master, 1024)
# Stream data to the stdout's fd of 0
os.write(0, buf)
if log is not None:
log.write(buf)
except Exception:
ctx.abort()
else:
return p.returncode
finally:
# cleanup
os.close(master)
os.close(slave)
def start_connection(play_context, variables, task_uuid):
'''
Starts the persistent connection
'''
candidate_paths = [C.ANSIBLE_CONNECTION_PATH or os.path.dirname(sys.argv[0])]
candidate_paths.extend(os.environ.get('PATH', '').split(os.pathsep))
for dirname in candidate_paths:
ansible_connection = os.path.join(dirname, 'ansible-connection')
if os.path.isfile(ansible_connection):
display.vvvv("Found ansible-connection at path {0}".format(ansible_connection))
break
else:
raise AnsibleError("Unable to find location of 'ansible-connection'. "
"Please set or check the value of ANSIBLE_CONNECTION_PATH")
env = os.environ.copy()
env.update({
# HACK; most of these paths may change during the controller's lifetime
# (eg, due to late dynamic role includes, multi-playbook execution), without a way
# to invalidate/update, ansible-connection won't always see the same plugins the controller
# can.
'ANSIBLE_BECOME_PLUGINS': become_loader.print_paths(),
'ANSIBLE_CLICONF_PLUGINS': cliconf_loader.print_paths(),
'ANSIBLE_COLLECTIONS_PATH': to_native(os.pathsep.join(AnsibleCollectionConfig.collection_paths)),
'ANSIBLE_CONNECTION_PLUGINS': connection_loader.print_paths(),
'ANSIBLE_HTTPAPI_PLUGINS': httpapi_loader.print_paths(),
'ANSIBLE_NETCONF_PLUGINS': netconf_loader.print_paths(),
'ANSIBLE_TERMINAL_PLUGINS': terminal_loader.print_paths(),
})
python = sys.executable
master, slave = pty.openpty()
p = subprocess.Popen(
[python, ansible_connection, to_text(os.getppid()), to_text(task_uuid)],
stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env
)
os.close(slave)
# We need to set the pty into noncanonical mode. This ensures that we
# can receive lines longer than 4095 characters (plus newline) without
# truncating.
old = termios.tcgetattr(master)
new = termios.tcgetattr(master)
new[3] = new[3] & ~termios.ICANON
try:
termios.tcsetattr(master, termios.TCSANOW, new)
write_to_file_descriptor(master, variables)
write_to_file_descriptor(master, play_context.serialize())
(stdout, stderr) = p.communicate()
finally:
termios.tcsetattr(master, termios.TCSANOW, old)
os.close(master)
if p.returncode == 0:
result = json.loads(to_text(stdout, errors='surrogate_then_replace'))
else:
try:
result = json.loads(to_text(stderr, errors='surrogate_then_replace'))
except getattr(json.decoder, 'JSONDecodeError', ValueError):
# JSONDecodeError only available on Python 3.5+
result = {'error': to_text(stderr, errors='surrogate_then_replace')}
if 'messages' in result:
for level, message in result['messages']:
if level == 'log':
display.display(message, log_only=True)
elif level in ('debug', 'v', 'vv', 'vvv', 'vvvv', 'vvvvv', 'vvvvvv'):
getattr(display, level)(message, host=play_context.remote_addr)
else:
if hasattr(display, level):
getattr(display, level)(message)
else:
display.vvvv(message, host=play_context.remote_addr)
if 'error' in result:
if play_context.verbosity > 2:
if result.get('exception'):
msg = "The full traceback is:\n" + result['exception']
display.display(msg, color=C.COLOR_ERROR)
raise AnsibleError(result['error'])
return result['socket_path']
def execute(self, context, args, in_opt_format=None, out_opt_format=None):
# This function is complex. There are two major variables. First,
# are we on Unix or Windows? This is effectively determined by
# pty_available, though I suppose some Unixes might not have ptys.
# Second, out_opt_format tells us whether we want to stream the
# output as lines (out_opt_format is None), or as unbuffered byte chunks
# (determined by bytearray/chunked). There is also a special hack
# x-filedescriptor/special where we pass along a file descriptor from
# the subprocess; this is used in unicode.py to directly read the output.
using_pty_out = pty_available and (out_opt_format not in (
None, 'x-unix-pipe-file-object/special'))
using_pty_in = pty_available and (in_opt_format is None) and \
context.input_is_first and hasattr(context.input, 'connect')
_logger.debug("using pty in: %s out: %s", using_pty_in, using_pty_out)
# TODO - we need to rework things so that we allocate only one pty per pipeline.
# In the very common case of exactly one command, this doesn't matter, but
# allocating two ptys will probably bite us in odd ways if someone does create
# a pipeline. Maybe have a context.request_pty() function?
if using_pty_in or using_pty_out:
# We create a pseudo-terminal to ensure that the subprocess is line-buffered.
# Yes, this is gross, but as far as I know there is no other way to
# control the buffering used by subprocesses.
(master_fd, slave_fd) = pty.openpty()
# Set the terminal to not do any processing; if you change this, you'll also
# need to update unicode.py most likely.
attrs = termios.tcgetattr(master_fd)
attrs[1] = attrs[1] & (~termios.OPOST)
termios.tcsetattr(master_fd, termios.TCSANOW, attrs)
_logger.debug("allocated pty fds %d %d", master_fd, slave_fd)
if using_pty_out:
stdout_target = slave_fd
else:
stdout_target = subprocess.PIPE
if context.input is None:
stdin_target = None
if using_pty_in:
stdin_target = slave_fd
elif in_opt_format == 'x-unix-pipe-file-object/special':
stdin_target = iter(context.input).next()
else:
stdin_target = subprocess.PIPE
_logger.debug("using stdin target: %r", stdin_target)
context.attribs['master_fd'] = master_fd
else:
_logger.debug(
"no pty available or non-chunked output, not allocating fds")
(master_fd, slave_fd) = (None, None)
stdout_target = subprocess.PIPE
if context.input is None:
stdin_target = None
elif in_opt_format == 'x-unix-pipe-file-object/special':
stdin_target = iter(context.input).next()
else:
stdin_target = subprocess.PIPE
_logger.debug("using stdin target: %r", stdin_target)
subproc_args = {
'bufsize': 0,
'stdin': stdin_target,
'stdout': stdout_target,
'stderr': subprocess.STDOUT,
'cwd': context.cwd
}
fs_encoding = sys.getfilesystemencoding()
stdin_encoding = sys.stdin.encoding
_logger.debug("recoding path to %r, args to %r", fs_encoding,
stdin_encoding)
# We need to encode_sentence arguments to the system encoding because subprocess.py won't do it for us.
if fs_encoding is not None:
args[0] = args[0].encode(fs_encoding)
if stdin_encoding is not None:
args[1:] = map(lambda x: x.encode(stdin_encoding), args[1:])
if is_windows():
subproc_args['universal_newlines'] = True
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
import win32con
startupinfo.wShowWindow = win32con.SW_HIDE
elif is_unix():
subproc_args['close_fds'] = True
# Support freedesktop.org startup notification
# http://standards.freedesktop.org/startup-notification-spec/startup-notification-latest.txt
if context.gtk_event_time:
env = dict(os.environ)
env['DESKTOP_STARTUP_ID'] = 'hotwire%d_TIME%d' % (
os.getpid(),
context.gtk_event_time,
)
subproc_args['env'] = env
def preexec():
os.setsid()
if using_pty_out and hasattr(termios, 'TIOCSCTTY'):
# Set our controlling TTY
fcntl.ioctl(1, termios.TIOCSCTTY, '')
# We ignore SIGHUP by default, because some broken programs expect that the lifetime
# of subcommands is tied to an open window, instead of until when the toplevel
# process exits.
signal.signal(signal.SIGHUP, signal.SIG_IGN)
subproc_args['preexec_fn'] = preexec
else:
assert (False)
subproc = subprocess.Popen(args, **subproc_args)
if not subproc.pid:
if master_fd is not None:
os.close(master_fd)
if slave_fd is not None:
os.close(slave_fd)
raise ValueError('Failed to execute %s' % (args[0], ))
context.attribs['pid'] = subproc.pid
if using_pty_in or using_pty_out:
os.close(slave_fd)
context.status_notify('pid %d' % (context.attribs['pid'], ))
if in_opt_format == 'x-unix-pipe-file-object/special':
stdin_target.close()
# If we were passed a file descriptor from another SysBuiltin, close it here;
# it's now owned by the child.
elif context.input:
if using_pty_in:
stdin_stream = BareFdStreamWriter(master_fd)
else:
stdin_stream = subproc.stdin
# FIXME hack - need to rework input streaming
if context.input_is_first and hasattr(context.input, 'connect'):
context.attribs['input_connected'] = True
context.input.connect(self.__on_input, stdin_stream)
else:
MiniThreadPool.getInstance().run(self.__inputwriter,
args=(context.input,
stdin_stream))
if using_pty_out:
stdout_read = None
stdout_fd = master_fd
else:
stdout_read = subproc.stdout
stdout_fd = subproc.stdout.fileno()
if out_opt_format is None:
for line in SysBuiltin.__unbuffered_readlines(stdout_read):
yield line
elif out_opt_format == 'bytearray/chunked':
try:
for buf in SysBuiltin.__unbuffered_read_pipe(
stream=stdout_read, fd=stdout_fd):
yield buf
except OSError, e:
pass
# INPUT = device for multiplexed data
# INPUT_BAUD = input baud rate
# OUTPUTS = devices for de-multiplexed data ["/dev/ttyADDRESS0", /dev/ttyADDRESS1", ...]
# output baud rates don't matter since they are virtual pty devices that work on any baud rate
INPUT = "/dev/ttyTHS2"
INPUT_BAUD = 57600
OUTPUTS = [
"/dev/ttyNC0", "/dev/gps0", "/dev/roboclaw0", "/dev/ttyNC1", "/dev/imu0"
]
output_devices = []
input_ser = serial.Serial(INPUT, INPUT_BAUD)
for output_symlink in OUTPUTS:
master, slave = pty.openpty()
master_name = os.ttyname(master)
slave_name = os.ttyname(slave)
if os.path.lexists(output_symlink):
os.unlink(output_symlink)
print("unlinked %s" % output_symlink)
os.symlink(slave_name, output_symlink)
print("created symlink %s -> %s" % (slave_name, output_symlink))
output_devices.append({
"master": master,
"slave": slave,
"symlink": output_symlink,
})
def measure(evl, argl, equations, evnames):
warned = False
all_events = gen_events(evl)
## use a pty because perf doesn't do enough fflush() otherwise
outp, inp = pty.openpty()
logfile = "ulog.%d" % (os.getpid())
run = PerfRun()
run.execute([perf, "stat", "--log-fd",
"%d" % (inp), "-e", all_events] + argl, logfile, evl)
prev_timestamp = None
evp = defaultdict(list)
res = defaultdict(list)
socket = ""
try:
if args.mock:
f = open(logfile, 'r')
else:
f = os.fdopen(outp, 'r')
os.close(inp)
while True:
try:
# force line-by-line buffering
l = f.readline()
if not l:
break
except KeyboardInterrupt, exceptions.IOError:
break
l = l.strip()
dbg("perf", l)
if l.startswith('#') or l == "":
continue
if per_socket:
ts, socket, _, rest = l.split(",", 3)
l = ts + "," + rest
# uncore// contains commas!
m = re.match(r"([0-9.]+),([0-9]+|<.*>),(.*)$", l)
if not m:
print "PERF-UNREADABLE", l,
continue
timestamp = m.group(1)
if timestamp != prev_timestamp:
if per_socket and not args.quiet:
warned = check_per_socket(res.keys(), warned)
if evp:
num = len(res)
gen_res(evl * num, concat(res),
concat(evp), equations * num,
gennames(evnames, res.keys()), timestamp)
res = defaultdict(list)
evp = defaultdict(list)
prev_timestamp = timestamp
r = m.group(2)
if r.startswith("<"):
if r in perf_errors:
r = perf_errors[r]
else:
r = "#NA"
res[socket].append(r)
evp[socket].append(m.group(3))
f.close()
if args.mock:
os.remove(logfile)
def run(
cmd: Union[str, List[str]],
print_error=True,
asynchronous=False,
stdin=False,
stderr=subprocess.STDOUT,
outfile=None,
env_vars: Optional[Dict[AnyStr, AnyStr]] = None,
inherit_cwd=False,
inherit_env=True,
tty=False,
shell=True,
cwd: str = None,
) -> Union[str, subprocess.Popen]:
LOG.debug("Executing command: %s", cmd)
env_dict = os.environ.copy() if inherit_env else {}
if env_vars:
env_dict.update(env_vars)
env_dict = {k: to_str(str(v)) for k, v in env_dict.items()}
if isinstance(cmd, list):
# See docs of subprocess.Popen(...):
# "On POSIX with shell=True, the shell defaults to /bin/sh. If args is a string,
# the string specifies the command to execute through the shell. [...] If args is
# a sequence, the first item specifies the command string, and any additional
# items will be treated as additional arguments to the shell itself."
# Hence, we should *disable* shell mode here to be on the safe side, to prevent
# arguments in the cmd list from leaking into arguments to the shell itself. This will
# effectively allow us to call run(..) with both - str and list - as cmd argument, although
# over time we should move from "cmd: Union[str, List[str]]" to "cmd: List[str]" only.
shell = False
if tty:
asynchronous = True
stdin = True
try:
if inherit_cwd and not cwd:
cwd = os.getcwd()
if not asynchronous:
if stdin:
return subprocess.check_output(cmd,
shell=shell,
stderr=stderr,
env=env_dict,
stdin=subprocess.PIPE,
cwd=cwd)
output = subprocess.check_output(cmd,
shell=shell,
stderr=stderr,
env=env_dict,
cwd=cwd)
return output.decode(config.DEFAULT_ENCODING)
stdin_arg = subprocess.PIPE if stdin else None
stdout_arg = open(outfile, "ab") if isinstance(outfile,
str) else outfile
stderr_arg = stderr
if tty:
# Note: leave the "pty" import here (not supported in Windows)
import pty
master_fd, slave_fd = pty.openpty()
stdin_arg = slave_fd
stdout_arg = stderr_arg = None
# start the actual sub process
kwargs = {}
if is_linux() or is_mac_os():
kwargs["start_new_session"] = True
process = subprocess.Popen(
cmd,
shell=shell,
stdin=stdin_arg,
bufsize=-1,
stderr=stderr_arg,
stdout=stdout_arg,
env=env_dict,
cwd=cwd,
**kwargs,
)
if tty:
# based on: https://stackoverflow.com/questions/41542960
def pipe_streams(*args):
while process.poll() is None:
r, w, e = select.select([sys.stdin, master_fd], [], [])
if sys.stdin in r:
d = os.read(sys.stdin.fileno(), 10240)
os.write(master_fd, d)
elif master_fd in r:
o = os.read(master_fd, 10240)
if o:
os.write(sys.stdout.fileno(), o)
FuncThread(pipe_streams).start()
return process
except subprocess.CalledProcessError as e:
if print_error:
print("ERROR: '%s': exit code %s; output: %s" %
(cmd, e.returncode, e.output))
sys.stdout.flush()
raise e
parser = argparse.ArgumentParser(description="LiteX JTAG UART bridge tool")
parser.add_argument("--config",
default="openocd_xc7_ft2232.cfg",
help="OpenOCD config file")
parser.add_argument("--telnet-port",
default="20000",
help="OpenOCD telnet port")
args = parser.parse_args()
def openocd_jtag_telnet():
prog = OpenOCD(args.config)
prog.stream(int(args.telnet_port))
m, s = pty.openpty()
print("LiteX JTAG UART created: {}".format(os.ttyname(s)))
openocd_jtag_telnet_thread = threading.Thread(target=openocd_jtag_telnet)
openocd_jtag_telnet_thread.start()
time.sleep(1)
t = telnetlib.Telnet("localhost", int(args.telnet_port))
def pty2telnet(m):
while True:
r = os.read(m, 1)
t.write(r)
if r == bytes("\n".encode("utf-8")):
def mkpty():
master, slave = pty.openpty()
slaveName = os.ttyname(slave)
print '\nslave device names: ', slaveName
return master, slaveName
def _start_connection(self, variables):
'''
Starts the persistent connection
'''
master, slave = pty.openpty()
python = sys.executable
def find_file_in_path(filename):
# Check $PATH first, followed by same directory as sys.argv[0]
paths = os.environ['PATH'].split(os.pathsep) + [os.path.dirname(sys.argv[0])]
for dirname in paths:
fullpath = os.path.join(dirname, filename)
if os.path.isfile(fullpath):
return fullpath
raise AnsibleError("Unable to find location of '%s'" % filename)
p = subprocess.Popen(
[python, find_file_in_path('ansible-connection'), to_text(os.getppid())],
stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdin = os.fdopen(master, 'wb', 0)
os.close(slave)
# Need to force a protocol that is compatible with both py2 and py3.
# That would be protocol=2 or less.
# Also need to force a protocol that excludes certain control chars as
# stdin in this case is a pty and control chars will cause problems.
# that means only protocol=0 will work.
src = cPickle.dumps(self._play_context.serialize(), protocol=0)
stdin.write(src)
stdin.write(b'\n#END_INIT#\n')
src = cPickle.dumps(variables, protocol=0)
stdin.write(src)
stdin.write(b'\n#END_VARS#\n')
stdin.flush()
(stdout, stderr) = p.communicate()
stdin.close()
if p.returncode == 0:
result = json.loads(to_text(stdout, errors='surrogate_then_replace'))
else:
try:
result = json.loads(to_text(stderr, errors='surrogate_then_replace'))
except getattr(json.decoder, 'JSONDecodeError', ValueError):
# JSONDecodeError only available on Python 3.5+
result = {'error': to_text(stderr, errors='surrogate_then_replace')}
if 'messages' in result:
for msg in result.get('messages'):
display.vvvv('%s' % msg, host=self._play_context.remote_addr)
if 'error' in result:
if self._play_context.verbosity > 2:
if result.get('exception'):
msg = "The full traceback is:\n" + result['exception']
display.display(msg, color=C.COLOR_ERROR)
raise AnsibleError(result['error'])
return result['socket_path']
def __init__(self,
testload,
speed=4800,
databits=8,
parity='N',
stopbits=1,
progress=None):
super(FakePTY, self).__init__(testload, progress)
# Allow Serial: header to be overridden by explicit speed.
if self.testload.serial:
(speed, databits, parity, stopbits) = self.testload.serial
self.speed = speed
baudrates = {
0: termios.B0,
50: termios.B50,
75: termios.B75,
110: termios.B110,
134: termios.B134,
150: termios.B150,
200: termios.B200,
300: termios.B300,
600: termios.B600,
1200: termios.B1200,
1800: termios.B1800,
2400: termios.B2400,
4800: termios.B4800,
9600: termios.B9600,
19200: termios.B19200,
38400: termios.B38400,
57600: termios.B57600,
115200: termios.B115200,
230400: termios.B230400,
}
(self.fd, self.slave_fd) = pty.openpty()
self.byname = os.ttyname(self.slave_fd)
os.chmod(
self.byname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP
| stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH)
(iflag, oflag, cflag, lflag, ispeed, ospeed,
cc) = termios.tcgetattr(self.slave_fd)
cc[termios.VMIN] = 1
cflag &= ~(termios.PARENB | termios.PARODD | termios.CRTSCTS)
cflag |= termios.CREAD | termios.CLOCAL
iflag = oflag = lflag = 0
iflag &= ~(termios.PARMRK | termios.INPCK)
cflag &= ~(termios.CSIZE | termios.CSTOPB | termios.PARENB
| termios.PARODD)
if databits == 7:
cflag |= termios.CS7
else:
cflag |= termios.CS8
if stopbits == 2:
cflag |= termios.CSTOPB
# Warning: attempting to set parity makes Fedora lose its cookies
if parity == 'E':
iflag |= termios.INPCK
cflag |= termios.PARENB
elif parity == 'O':
iflag |= termios.INPCK
cflag |= termios.PARENB | termios.PARODD
ispeed = ospeed = baudrates[speed]
try:
termios.tcsetattr(self.slave_fd, termios.TCSANOW,
[iflag, oflag, cflag, lflag, ispeed, ospeed, cc])
except termios.error:
raise TestLoadError("error attempting to set serial mode to %s "
" %s%s%s" %
(speed, databits, parity, stopbits))
def main():
args = docopt(__doc__, version=__version__)
if args.pop('--debug-trace', False):
args['--debug'] = "NOTSET"
log.basicConfig(level=getattr(log, args['--debug'], log.INFO),
format='%(levelname)s: %(message)s')
log.debug(args)
# Get compressed sizes
zips = {}
for fn in args['<zipfiles>']:
info = subprocess.check_output(["7z", "l", fn]).strip()
finfo = RE_SCN.findall(info)
# builtin test: last line should be total sizes
log.debug(finfo)
totals = map(int, finfo[-1][:2])
# log.debug(totals)
for s in range(2):
assert (sum(map(int, (inf[s] for inf in finfo[:-1]))) == totals[s])
fcomp = dict((n, int(c if args['--compressed'] else u))
for (u, c, n) in finfo[:-1])
# log.debug(fcomp)
# zips : {'zipname' : {'filename' : int(size)}}
zips[fn] = fcomp
# Extract
cmd7zx = ["7z", "x", "-bd"]
if args['--yes']:
cmd7zx += ["-y"]
log.info("Extracting from {:d} file(s)".format(len(zips)))
with tqdm(total=sum(sum(fcomp.values()) for fcomp in zips.values()),
unit="B",
unit_scale=True) as tall:
for fn, fcomp in zips.items():
md, sd = pty.openpty()
ex = subprocess.Popen(
cmd7zx + [fn],
bufsize=1,
stdout=md, # subprocess.PIPE,
stderr=subprocess.STDOUT)
os.close(sd)
with io.open(md, mode="rU", buffering=1) as m:
with tqdm(total=sum(fcomp.values()),
disable=len(zips) < 2,
leave=False,
unit="B",
unit_scale=True) as t:
while True:
try:
l_raw = m.readline()
except IOError:
break
l = l_raw.strip()
if l.startswith("Extracting"):
exname = l.lstrip("Extracting").lstrip()
s = fcomp.get(exname, 0) # 0 is likely folders
t.update(s)
tall.update(s)
elif l:
if not any(
l.startswith(i)
for i in ("7-Zip ", "p7zip Version ",
"Everything is Ok", "Folders: ",
"Files: ", "Size: ",
"Compressed: ")):
if l.startswith("Processing archive: "):
if not args['--silent']:
t.write(
t.format_interval(t.start_t -
tall.start_t) +
' ' +
l.lstrip("Processing archive: "))
else:
t.write(l)
ex.wait()
def main():
args = argopt(__doc__, version=__version__).parse_args()
if args.debug_trace:
args.debug = "NOTSET"
logging.basicConfig(level=getattr(logging, args.debug, logging.INFO),
format='%(levelname)s:%(message)s')
log = logging.getLogger(__name__)
log.debug(args)
# Get compressed sizes
zips = {}
for fn in args.zipfiles:
info = subprocess.check_output(["7z", "l", fn]).strip() # nosec
finfo = RE_SCN.findall(info) # size|compressed|name
# builtin test: last line should be total sizes
log.debug(finfo)
totals = map(int, finfo[-1][:2])
# log.debug(totals)
for s in range(2): # size|compressed totals
totals_s = sum(map(int, (inf[s] for inf in finfo[:-1])))
if totals_s != totals[s]:
log.warn("%s: individual total %d != 7z total %d",
fn, totals_s, totals[s])
fcomp = {n: int(c if args.compressed else u) for (u, c, n) in finfo[:-1]}
# log.debug(fcomp)
# zips : {'zipname' : {'filename' : int(size)}}
zips[fn] = fcomp
# Extract
cmd7zx = ["7z", "x", "-bd"]
if args.yes:
cmd7zx += ["-y"]
log.info("Extracting from %d file(s)", len(zips))
with tqdm(total=sum(sum(fcomp.values()) for fcomp in zips.values()),
unit="B", unit_scale=True) as tall:
for fn, fcomp in zips.items():
md, sd = pty.openpty()
ex = subprocess.Popen(
cmd7zx + [fn],
bufsize=1,
stdout=md, # subprocess.PIPE,
stderr=subprocess.STDOUT) # nosec
os.close(sd)
with io.open(md, mode="rU", buffering=1) as m:
with tqdm(total=sum(fcomp.values()), disable=len(zips) < 2,
leave=False, unit="B", unit_scale=True) as t:
if not hasattr(t, "start_t"): # disabled
t.start_t = tall._time()
while True:
try:
l_raw = m.readline()
except IOError:
break
ln = l_raw.strip()
if ln.startswith("Extracting"):
exname = ln.lstrip("Extracting").lstrip()
s = fcomp.get(exname, 0) # 0 is likely folders
t.update(s)
tall.update(s)
elif ln:
if not any(
ln.startswith(i)
for i in ("7-Zip ", "p7zip Version ",
"Everything is Ok", "Folders: ",
"Files: ", "Size: ", "Compressed: ")):
if ln.startswith("Processing archive: "):
if not args.silent:
t.write(t.format_interval(
t.start_t - tall.start_t) + ' ' +
ln.replace("Processing archive: ", ""))
else:
t.write(ln)
ex.wait()
def make_slave_pty():
master_pty, slave_pty = pty.openpty()
yield slave_pty
os.close(slave_pty)
os.close(master_pty)
def run(self):
#Opening pseudo terminal
mfd, sfd = pty.openpty()
#Measure system before run
extend_psutil_measure = __psutil_measure__()
self.logger.debug(
("MULTILINE:\n{1} Running" + " process:\n {0}\n {1}").format(
self.cmd, __format_span_next_line__("*_*", self.config)))
#start process
popen_obj = Popen(self.cmd,
stdin=sfd,
stdout=sfd,
stderr=sfd,
shell=True)
self.logger.info(
("\n{span:-^80}\n" + "{message: ^80}\n{span:-^80}\n").format(
span="PROCESS", message=self.cmd))
#calculate stoptimes
general_time_out = self.__stoptime__(self.general_timeout)
ready_time_out = self.__stoptime__(self.ready_timeout)
response_time_out = self.__stoptime__(self.response_timeout)
#<Play wiht pids:>
self.logger.debug("Python process pid is: %i" % os.getpid())
#Get real pid of process by sh pid
# - becouse we ran this process in shell mode
# and we need to find all children of this process (must be only one of cource).
real_pid = self.get_children_of_pid(popen_obj.pid)
process_dubbed = False
if real_pid != popen_obj.pid:
process_dubbed = True
self.pids = [real_pid, popen_obj.pid]
self.logger.info("Shell /bin/sh pid: %i" % popen_obj.pid)
self.logger.info("Sub process pid is: %i" % real_pid)
else:
self.pids = [real_pid]
self.logger.info("Sub process pid is: %i" % real_pid)
#</:Play wiht pids>
#preparing result object
self.result = self.__result__(
real_pid, self.logger, extend_psutil_measure=extend_psutil_measure)
communicator = None
jogging_status = [".", "..", "...", ".."]
jogging_status_idx = 0
jogging = False
while True:
self.logger.debug(
("MULTILINE:\n{1}\nCurrent time is:" +
" {4}\n{1}Times for out:\n\t\t -" +
" response_time_out = {0}\n\t\t -" +
" general_time_out={2}\n\t\t - " +
"ready_time_out={3}\n {1}").format(
response_time_out,
__format_span_next_line__("*_*", self.config),
general_time_out, ready_time_out, float(time.time())))
#<Select result from pty by file description>
self.logger.debug("Before select")
#Calculate time that we spent for select response
time_spend_for_respons = float(time.time())
r, w, e = select([mfd], [], [mfd], self.select_timeout)
time_spend_for_respons = float(
time.time()) - time_spend_for_respons
self.logger.debug("After select")
#</Select result from pty by file description>
#Update status of process
popen_obj.poll()
#<system measure of current process>
#Update measure
self.result.measure.update()
if hasattr(self.result.measure, "cpu_usage_average"):
self.logger.debug("Current cpu usage average is : %f" %
(self.result.measure.cpu_usage_average))
#</system measure of current process>
self.result.return_code = popen_obj.returncode
#recive cmd`s from queue in thread mode
if (self.thread_mode):
if not self.queue.empty():
queue_command = self.queue.get()
if "KILL" in queue_command.upper():
self.logger.debug("KILL signal recived")
self.stop()
self.queue.task_done()
elif ("jogging" in queue_command):
jogging = True
if jogging:
if jogging_status_idx < (len(jogging_status) - 1):
jogging_status_idx += 1
else:
jogging_status_idx = 0
self.logger.info(
"Pid %s work in thread%s" %
(real_pid, jogging_status[jogging_status_idx]))
communicator_decoded = ""
if mfd in r:
#self.logger.debug("Return code: %s " % str(self.result.return_code))
#Reading data from pty
communicator = os.read(mfd, 10240)
#communicator = communicator.replace(b'\x08', b'') ????????????
#Sometimes event problem with decoding of unrecognize simbols
# because screen out from process came in two parts
# and simbols starting at the end of one part ended at the begin of other -
# - in two differents communicators variable (divided in time)
# todo: need first collect than decode
try:
communicator_decoded = communicator.decode(
self.config.common.default_codepage)
# self.logger.debug("Decoded std(err)out: %s" % (repr(communicator_decoded)))
except UnicodeDecodeError as ude:
self.logger.debug(
"Durring decoding result to %s raised error: %s\t-\t Try to workaroud."
% (self.config.common.default_codepage, ude))
start_pos = ude.args[2]
end_pos = ude.args[3]
bad_bit = communicator[start_pos:end_pos]
self.logger.debug(
"Bad simbol is %s, between %i and %i posions.\t Will be trim part: %s"
% (bad_bit, start_pos, end_pos,
communicator[start_pos:]))
communicator_trim = communicator[:start_pos]
communicator_decoded = communicator_trim.decode(
self.config.common.default_codepage)
#<Sudo mechanism:>
# !ATTENTION!: use -S option with sudo for recive password through stdin
if "password for" in communicator_decoded:
self.logger.debug("Asking for root password: %s" %
communicator_decoded)
if hasattr(self.config.common, "user_password"):
pswd = bytes(
"%s\n" % (self.config.common.user_password),
self.config.common.default_codepage)
if "Sorry, try again" in communicator_decoded:
self.logger.warning(
"!!ERROR: Password for sudo user %s is uncorrect %s "
% (self.config.common.user_name,
self.config.common.user_password))
self.result.errors_out_decoded = communicator_decoded
self.logger.debug("Password set to %s" % (repr(pswd)))
os.write(mfd, pswd)
continue
else:
self.logger.info(
"Sudo password is not provide! Stop process and exit..."
)
self.result.errors_out_decoded = "Sudo password is not provide!"
if (self.thread_mode):
self.stop()
#</Sudo mechanism>
#Check if process returned more than 0 meaning error
if (self.result.return_code
is not None) and (self.result.return_code > 0):
self.logger.warning("!!ERROR: Returned non zero! %s " %
(communicator_decoded))
self.result.errors_out_decoded = communicator_decoded
self.result.errors_out_bytes = communicator
#else work with result
else:
self.logger.debug("\t++: %s " % communicator_decoded)
response_time_out = self.__stoptime__(
self.response_timeout)
self.result.std_out_decoded += communicator_decoded
self.result.std_out_bytes += communicator
#<parce output by regular exprasion>
if (len(communicator_decoded) > 0) and (self.regexp_pars
is not None):
self.logger.debug(
"regexp_pars is not none. We are starting parsing standart output to class field..."
)
for key, value in self.regexp_pars.items():
try:
#self.logger.debug("Commu decode repr: %s" % (repr(communicator_decoded)) )
re_search = re.search(value,
communicator_decoded,
re.MULTILINE)
if re_search is not None:
setattr(self.result.screen_capture, key,
re_search.group(key))
self.logger.debug(
"Great we found %s it is %s" %
(key, re_search.group(key)))
else:
self.logger.debug(
"!!WARNING: Couldn't find %s" % key)
except IndexError:
self.logger.debug(
"!!WARNING: Output hasn't this group of data"
)
#</parce output by regular exprasion>
#Word after that we will be ready to communicate or just stop(kill or something like this) this process
if (self.wait_word) and (self.wait_word
in communicator_decoded) and (
ready_time_out < time.time()):
self.logger.debug("Catch wait word!")
# - thread mode
if (self.thread_mode):
if (not jogging):
#<work with thread>
self.logger.debug(
"\t-thread mode case starting preparing communicate..."
)
#queue_get = self.queue.get()
self.logger.debug(
"\t-thread: %s, recived %s" %
(str(self.id), queue_command[::-1]))
self.queue.task_done()
#</work with thread>
# - single mode
else:
command = shlex.split(
self.stop_cmd.format(
pids=" ".join(
[str(pi) for pi in self.pids]),
result=self.result
#port=self.result.screen_capture.esrv_port if hasattr(self.result.screen_capture, 'esrv_port'} else ""
))
self.logger.debug(
"\t\t-single mode case starting stop procedure by the command: %s"
% (" ".join(command)))
killer_return = run(command,
stdout=PIPE,
stderr=PIPE)
if killer_return.returncode > 0:
#softly exit not immplimented
raise Exception(
"Process stop errors! Return more than zerro. Error: %s "
% (self.result.stderr))
#<Timeout predicates>
#General time out exit
if (general_time_out < time.time()):
self.result.errors_out_decoded += "General timeout pass out! pass %f sec" % (
general_time_out)
self.logger.warning(self.result.errors_out_decoded)
if (self.thread_mode):
if (self.thread_mode):
try:
self.queue.task_done()
except:
pass
self.stop()
self.result.return_code = 1
break
#Response time out exit
if (response_time_out < time.time()):
self.result.errors_out_decoded += "Wait response timeout pass out! pass %f sec" % (
response_time_out)
self.logger.warning(self.result.errors_out_decoded)
if (self.thread_mode):
if (self.thread_mode):
try:
self.queue.task_done()
except:
pass
self.stop()
self.result.return_code = 1
break
#</Timeout predicates>
self.logger.debug("self.result.return_code: '%s' " %
(str(self.result.return_code)))
#<error predicat>
if (self.result.return_code
is not None) and (self.result.return_code > 0):
self.logger.debug(
"A-a-a-a! its returned more than zerro!! Stopping...")
if (len(self.result.errors_out_decoded) < 1):
if (communicator_decoded in locals().keys()
and communicator_decoded is not None):
error_descr = "Unrecognized error:\n %s" % (
communicator_decoded)
else:
error_descr = "Unrecognized error."
self.result.errors_out_decoded = error_descr
if (self.thread_mode):
try:
self.queue.task_done()
except:
pass
self.stop()
break
#</error predicat>
#if return zerro or below zerro meaning that we successfully done
elif (self.result.return_code is not None):
if (self.thread_mode):
self.queue.task_done()
break
self.logger.debug("{1} Finish prompt line {0} ...\n {1}".format(
self.cmd, __format_span_next_line__("*_*", self.config)))
return self.result
def call_command(command, command_timeout=None, output_timeout=None):
"""Call the given command with optional timeouts.
This function calls the given command in a shell environment and applies a
timeout on the whole command as well as a timeout on the stdout/stderr
streams.
The function returns the tuple (return code, stdout, stderr) with stdout
and stderr as the byte buffers. To convert them to strings, use the
'decode(encoding="utf-8", errors="ignore")' function.
The return code will be -1000 for a command timeout and -2000 for
a stdout timeout.
Args:
command (str): The command to execute.
command_timeout (float): The timeout of the whole command execution in seconds.
output_timeout (float): The timeout of stdout/stderr outputs.
Returns:
The tuple (return_code, stdout_buffer, stderr_buffer) with the return code
of the command (or -1000 on a command timeout resp. -2000 on a stdout timeout)
and the stdout/stderr buffers as byte arrays.
"""
LOG.debug(
"Execute command '%s' in shell environment with command timeout of %s seconds and "
"output timeout of %s seconds.", command, command_timeout,
output_timeout)
stdout_r, stdout_w = pty.openpty()
stderr_r, stderr_w = pty.openpty()
proc = subprocess.Popen(command,
bufsize=0,
shell=True,
stdout=stdout_w,
stderr=stderr_w)
os.close(stdout_w)
os.close(stderr_w)
output_start_time = time.time()
command_start_time = output_start_time
command_timeout_occured = False
output_timeout_occured = False
buffer_size = 1024
stdout_buffer = b''
stderr_buffer = b''
def read_buffer(read_fd, output_stream):
buffer = b''
try:
buffer = os.read(read_fd, buffer_size)
except OSError as exception:
if exception.errno != errno.EIO:
raise
else:
output_stream.buffer.write(buffer)
output_stream.buffer.flush()
return buffer
while proc.poll() is None:
# Since the select call can't detect whether the process exited, we use a
# small timeout on the select call (one second) and check afterwards for
# a timeout on the stdout/stderr streams
streams_ready = select.select([stdout_r, stderr_r], [], [], 1.0)[0]
if streams_ready:
if stdout_r in streams_ready:
stdout_buffer += read_buffer(stdout_r, sys.stdout)
if stderr_r in streams_ready:
stderr_buffer += read_buffer(stderr_r, sys.stderr)
output_start_time = time.time()
elif proc.poll() is None:
if output_timeout and (
(time.time() - output_start_time) >= output_timeout):
LOG.debug(
"No stdout/stderr output received within %d seconds!",
(time.time() - output_start_time))
output_timeout_occured = True
proc.kill()
break
if command_timeout and (
(time.time() - command_start_time) >= command_timeout):
LOG.debug("Timeout occured after %d seconds!",
(time.time() - command_start_time))
command_timeout_occured = True
proc.kill()
break
# To cleanup any pending resources
proc.wait()
os.close(stdout_r)
os.close(stderr_r)
sys.stdout.buffer.flush()
sys.stderr.buffer.flush()
return_code = proc.returncode
if command_timeout_occured:
return_code = -1000
elif output_timeout_occured:
return_code = -2000
LOG.debug("Command '%s' finished with return code %d.", command,
return_code)
return (return_code, stdout_buffer, stderr_buffer)
def exec_command(self,
cmd,
tmp_path,
sudo_user,
sudoable=False,
executable='/bin/sh',
in_data=None):
''' run a command on the remote host '''
ssh_cmd = self._password_cmd()
ssh_cmd += ["ssh", "-C"]
if not in_data:
ssh_cmd += ["-tt"]
if utils.VERBOSITY > 3:
ssh_cmd += ["-vvv"]
else:
ssh_cmd += ["-q"]
ssh_cmd += self.common_args
if self.ipv6:
ssh_cmd += ['-6']
ssh_cmd += [self.host]
if not self.runner.sudo or not sudoable:
if executable:
ssh_cmd.append(executable + ' -c ' + pipes.quote(cmd))
else:
ssh_cmd.append(cmd)
else:
sudocmd, prompt, success_key = utils.make_sudo_cmd(
sudo_user, executable, cmd)
ssh_cmd.append(sudocmd)
vvv("EXEC %s" % ssh_cmd, host=self.host)
not_in_host_file = self.not_in_host_file(self.host)
if C.HOST_KEY_CHECKING and not_in_host_file:
# lock around the initial SSH connectivity so the user prompt about whether to add
# the host to known hosts is not intermingled with multiprocess output.
fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_EX)
fcntl.lockf(self.runner.output_lockfile, fcntl.LOCK_EX)
# create process
if in_data:
# do not use pseudo-pty
p = subprocess.Popen(ssh_cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdin = p.stdin
else:
# try to use upseudo-pty
try:
# Make sure stdin is a proper (pseudo) pty to avoid: tcgetattr errors
master, slave = pty.openpty()
p = subprocess.Popen(ssh_cmd,
stdin=slave,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdin = os.fdopen(master, 'w', 0)
os.close(slave)
except:
p = subprocess.Popen(ssh_cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdin = p.stdin
self._send_password()
if self.runner.sudo and sudoable and self.runner.sudo_pass:
fcntl.fcntl(p.stdout, fcntl.F_SETFL,
fcntl.fcntl(p.stdout, fcntl.F_GETFL) | os.O_NONBLOCK)
sudo_output = ''
if in_data:
# no terminal => no prompt on output. process is waiting for sudo_pass
stdin.write(self.runner.sudo_pass + '\n')
while not sudo_output.endswith(
prompt) and success_key not in sudo_output:
rfd, wfd, efd = select.select([p.stdout], [], [p.stdout],
self.runner.timeout)
if p.stdout in rfd:
chunk = p.stdout.read()
if not chunk:
raise errors.AnsibleError(
'ssh connection closed waiting for sudo password prompt'
)
sudo_output += chunk
else:
stdout = p.communicate()
raise errors.AnsibleError(
'ssh connection error waiting for sudo password prompt'
)
if success_key not in sudo_output:
stdin.write(self.runner.sudo_pass + '\n')
fcntl.fcntl(p.stdout, fcntl.F_SETFL,
fcntl.fcntl(p.stdout, fcntl.F_GETFL) & ~os.O_NONBLOCK)
# We can't use p.communicate here because the ControlMaster may have stdout open as well
stdout = ''
stderr = ''
rpipes = [p.stdout, p.stderr]
if in_data:
try:
stdin.write(in_data)
stdin.close()
except:
raise errors.AnsibleError(
'SSH Error: data could not be sent to the remote host. Make sure this host can be reached over ssh'
)
while True:
rfd, wfd, efd = select.select(rpipes, [], rpipes, 1)
# fail early if the sudo password is wrong
if self.runner.sudo and sudoable and self.runner.sudo_pass:
incorrect_password = gettext.dgettext("sudo",
"Sorry, try again.")
if stdout.endswith("%s\r\n%s" % (incorrect_password, prompt)):
raise errors.AnsibleError('Incorrect sudo password')
if p.stdout in rfd:
dat = os.read(p.stdout.fileno(), 9000)
stdout += dat
if dat == '':
rpipes.remove(p.stdout)
if p.stderr in rfd:
dat = os.read(p.stderr.fileno(), 9000)
stderr += dat
if dat == '':
rpipes.remove(p.stderr)
# only break out if we've emptied the pipes, or there is nothing to
# read from and the process has finished.
if (not rpipes or not rfd) and p.poll() is not None:
break
# Calling wait while there are still pipes to read can cause a lock
elif not rpipes and p.poll() == None:
p.wait()
stdin.close(
) # close stdin after we read from stdout (see also issue #848)
if C.HOST_KEY_CHECKING and not_in_host_file:
# lock around the initial SSH connectivity so the user prompt about whether to add
# the host to known hosts is not intermingled with multiprocess output.
fcntl.lockf(self.runner.output_lockfile, fcntl.LOCK_UN)
fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_UN)
controlpersisterror = stderr.find(
'Bad configuration option: ControlPersist') != -1 or stderr.find(
'unknown configuration option: ControlPersist') != -1
if p.returncode != 0 and controlpersisterror:
raise errors.AnsibleError(
'using -c ssh on certain older ssh versions may not support ControlPersist, set ANSIBLE_SSH_ARGS="" (or ansible_ssh_args in the config file) before running again'
)
if p.returncode == 255 and in_data:
raise errors.AnsibleError(
'SSH Error: data could not be sent to the remote host. Make sure this host can be reached over ssh'
)
return (p.returncode, '', stdout, stderr)
def __init__(self, option_string=''):
self.read_loop = True
options = option_string.split(' ')
cmd = ['iperf'] + options
cmd_str = 'iperf ' + option_string
master, slave = pty.openpty()
self.process = Popen(cmd, stdout=slave, stderr=slave, close_fds=False)
self.stdout = os.fdopen(master, 'r', 10000)
# buffer which holds the iperf process output to read from
self.readbuf = ''
self.test_str = ''
self.test_end = False
# Prometheus export data
# helper variables to calculate the metrics
self.registry = CollectorRegistry()
#buckets = (0.1, 0.2, 0.5, 1, 2, 5, 7, 10, 20, 50, 70, 90, float("inf"))
self.prom_loss = Gauge('sonemu_packet_loss_percent',
'iperf packet loss (percent)', ['vnf_name'],
registry=self.registry)
self.prom_packets_loss = Gauge('sonemu_packets_loss_count',
'iperf packets lost (count)',
['vnf_name'],
registry=self.registry)
self.prom_packets_total = Gauge('sonemu_packets_total_count',
'iperf packets total (count)',
['vnf_name'],
registry=self.registry)
#buckets = (1, 9, 10, 11, 90, 100, 110, 900, 1000, 1100, float("inf"))
self.prom_bandwith = Gauge('sonemu_bandwith_Mbitspersec',
'iperf bandwith (Mbits/sec)', ['vnf_name'],
registry=self.registry)
#buckets = (0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 5, 10, float("inf"))
self.prom_jitter = Gauge('sonemu_jitter_ms',
'iperf jitter (ms)', ['vnf_name'],
registry=self.registry)
self.prom_bandwith.labels(vnf_name=vnf_name).set(float('nan'))
self.prom_loss.labels(vnf_name=vnf_name).set(float('nan'))
self.prom_packets_total.labels(vnf_name=vnf_name).set(float('nan'))
self.prom_packets_loss.labels(vnf_name=vnf_name).set(float('nan'))
self.prom_jitter.labels(vnf_name=vnf_name).set(float('nan'))
while True:
data = self.readline()
if data:
logging.info('stdout: {0}'.format(data))
self.parse_beginning_of_test(data)
self.parse_end_of_test(data)
if not self.test_end:
bw = self.parse_bandwith(data)
if not isnan(bw):
self.prom_bandwith.labels(vnf_name=vnf_name).set(bw)
else:
self.prom_bandwith.labels(vnf_name=vnf_name).set(bw)
# end of iperf test, no real measurement
continue
loss = self.parse_loss(data)
self.prom_loss.labels(vnf_name=vnf_name).set(loss)
lost, total = self.parse_packets(data)
if lost and total:
self.prom_packets_total.labels(
vnf_name=vnf_name).set(total)
self.prom_packets_loss.labels(
vnf_name=vnf_name).set(lost)
jitter = self.parse_jitter(data)
self.prom_jitter.labels(vnf_name=vnf_name).set(jitter)
else:
self.prom_loss.labels(vnf_name=vnf_name).set(float('nan'))
self.prom_jitter.labels(vnf_name=vnf_name).set(float('nan'))
pushadd_to_gateway(pushgateway,
job='sonemu-profile_sink',
registry=self.registry)
def __init__(self,
reactor,
executable,
args,
environment,
path,
proto,
uid=None,
gid=None,
usePTY=None):
"""
Spawn an operating-system process.
This is where the hard work of disconnecting all currently open
files / forking / executing the new process happens. (This is
executed automatically when a Process is instantiated.)
This will also run the subprocess as a given user ID and group ID, if
specified. (Implementation Note: this doesn't support all the arcane
nuances of setXXuid on UNIX: it will assume that either your effective
or real UID is 0.)
"""
if pty is None and not isinstance(usePTY, (tuple, list)):
# no pty module and we didn't get a pty to use
raise NotImplementedError(
"cannot use PTYProcess on platforms without the pty module.")
abstract.FileDescriptor.__init__(self, reactor)
_BaseProcess.__init__(self, proto)
if isinstance(usePTY, (tuple, list)):
masterfd, slavefd, ttyname = usePTY
else:
masterfd, slavefd = pty.openpty()
ttyname = os.ttyname(slavefd)
try:
self._fork(path,
uid,
gid,
executable,
args,
environment,
masterfd=masterfd,
slavefd=slavefd)
except:
if not isinstance(usePTY, (tuple, list)):
os.close(masterfd)
os.close(slavefd)
raise
# we are now in parent process:
os.close(slavefd)
fdesc.setNonBlocking(masterfd)
self.fd = masterfd
self.startReading()
self.connected = 1
self.status = -1
try:
self.proto.makeConnection(self)
except:
log.err()
registerReapProcessHandler(self.pid, self)
def get_ports_pty():
master, slave = pty.openpty()
s_name = os.ttyname(slave)
return master, slave, s_name
def startShell(self, mnopts=None):
"""Start a shell process for running commands."""
if self.shell:
error('shell is already running')
return
assert mnopts is None, 'mnopts not supported for DockerHost'
self.container = '%s-%s' % (self.prefix, self.name)
debug('Starting container %s with image "%s".' % (self.container, self.image))
self.kill(purge=True)
container_tmp_dir = os.path.join(os.path.abspath(self.tmpdir), 'tmp')
tmp_volume = container_tmp_dir + ':/tmp'
base_cmd = ["docker", "run", "-ti", "--privileged", "--entrypoint", "env",
"-h", self.name, "--name", self.container]
opt_args = ['--net=%s' % self.network]
env_vars = self.env_vars + ["TERM=dumb", "PS1=%s" % self.ps1]
env_args = reduce(operator.add, (['--env', var] for var in env_vars), [])
vol_args = reduce(operator.add, (['-v', var] for var in self.vol_maps), ['-v', tmp_volume])
image_args = [self.image, "bash", "--norc", "-is", "mininet:" + self.name]
cmd = base_cmd + opt_args + env_args + vol_args + image_args
self.master, self.slave = pty.openpty()
debug('docker command "%s", fd %d, fd %d' % (' '.join(cmd), self.master, self.slave))
try:
self.shell = self._popen(cmd, stdin=self.slave, stdout=self.slave, stderr=self.slave)
self.stdin = os.fdopen(self.master, 'r')
self.stdout = self.stdin
self.pollOut = select.poll() # pylint: disable=invalid-name
self.pollOut.register(self.stdout) # pylint: disable=no-member
self.outToNode[self.stdout.fileno()] = self # pylint: disable=no-member
self.pollIn = select.poll() # pylint: disable=invalid-name
self.pollIn.register(self.stdout, select.POLLIN) # pylint: disable=no-member
self.inToNode[self.stdin.fileno()] = self # pylint: disable=no-member
self.execed = False
self.lastCmd = None # pylint: disable=invalid-name
self.lastPid = None # pylint: disable=invalid-name
self.readbuf = ''
self.waiting = True
data = ''
while True:
data = self.read(1)
if data[-1] == self.ps1:
break
self.readbuf = ''
self.waiting = False
except Exception:
error('docker cmd: %s' % ' '.join(cmd))
if self.shell.returncode:
error('returncode: %d' % self.shell.returncode)
if self.shell:
self.shell.poll()
raise
self.pid = self.inspect_pid()
debug("Container %s created pid %s/%s." % (self.container, self.pid, self.shell.pid))
self.cmd('unset HISTFILE; stty -echo; set +m') # pylint: disable=no-member
def run(
cmd,
address=None,
service_type=None,
name=None,
user=None,
user_mode=os.getuid() != 0,
nice=None,
runtime_max_sec=None,
env=None,
extra=None,
cwd=None,
machine=None,
wait=False,
remain_after_exit=False,
collect=False,
raise_on_fail=False,
pty=None,
pty_master=None,
pty_path=None,
stdin=None,
stdout=None,
stderr=None,
_wait_polling=None,
slice_=None,
stop_cmd=None,
stop_post_cmd=None,
start_pre_cmd=None,
start_post_cmd=None,
):
"""
pystemd.run imitates systemd-run, but with a pythonic feel to it.
Options:
cmd: Array with the command to execute (absolute path only)
stop_cmd: Array with the command to execute on stop (absolute path only)
stop_post_cmd: Array with the command to execute after stop (absolute path only)
start_pre_cmd: Array with the command to execute on pre start (absolute path only)
start_post_cmd: Array with the command to execute on on post start (absolute path only)
address: A custom dbus socket address
service_type: Set the unit type, e.g. notify, oneshot. If you dont give a
value, the unit type will be whatever systemd thinks is the default.
name: Name of the unit. If not provided, it will be autogenerated.
user: Username to execute the command, defaults to current user.
user_mode: Equivalent to running `systemd-run --user`. Defaults to True
if current user id not root (uid = 0).
nice: Nice level to run the command.
runtime_max_sec: Set seconds before sending a sigterm to the process, if
the service does not die nicely, it will send a sigkill.
env: A dict with environment variables.
extra: If you know what you are doing, you can pass extra configuration
settings to the start_transient_unit method.
machine: Machine name to execute the command, by default we connect to
the host's dbus.
wait: Wait for command completion before returning control, defaults
to False.
remain_after_exit: If True, the transient unit will remain after cmd
has finished, also if true, this methods will return
pystemd.systemd1.Unit object. defaults to False and this method
returns None and the unit will be gone as soon as is done.
collect: Unload unit after it ran, even when failed.
raise_on_fail: Will raise a PystemdRunError is cmd exit with non 0
status code, it won't take affect unless you set wait=True,
defaults to False.
pty: Set this variable to True if you want a pty to be created. if you
pass a `machine`, the pty will be created in the machine. Setting
this value will ignore whatever you set in pty_master and pty_path.
pty_master: It has only meaning if you pass a pty_path also, this file
descriptor will be used to forward redirection to `stdin` and `stdout`
if no `stdin` or `stdout` is present, then this value does nothing.
pty_path: Setting this value will pass this pty_path to the created
process and will connect the process stdin, stdout and stderr to this
pty. by itself it only ensure that your process has a real pty that
can have ioctl operation over it. if you also pass a `pty_master`,
`stdin` and `stdout` the pty forwars is handle for you.
stdin: Specify a file descriptor for stdin. By default this is `None`
and your unit will not have a stdin. If you set pty = True, or set a
`pty_master` then that pty will be read and forwarded to this file
descriptor.
stdout: Specify a file descriptor for stdout. By default this is `None`
and your unit will not have a stdout. If you set pty = True, or set a
`pty_master` then that pty will be read and forwarded to this file
descriptor.
stderr: Specify a file descriptor for stderr. By default this is `None`
and your unit will not have a stderr.
slice_: the slice under you want to run the unit.
More info and examples in:
https://github.com/facebookincubator/pystemd/blob/master/_docs/pystemd.run.md
"""
def bus_factory():
if address:
return DBusAddress(x2char_star(address))
elif machine:
return DBusMachine(x2char_star(machine))
else:
return DBus(user_mode=user_mode)
name = x2char_star(name or "pystemd{}.service".format(uuid.uuid4().hex))
runtime_max_usec = (runtime_max_sec or 0) * 10**6 or runtime_max_sec
stdin, stdout, stderr = get_fno(stdin), get_fno(stdout), get_fno(stderr)
env = env or {}
unit_properties = {}
selectors = []
extra = extra or {}
start_cmd = x2cmdlist(cmd, False) + extra.pop(b"ExecStart", [])
stop_cmd = x2cmdlist(stop_cmd, False) + extra.pop(b"ExecStop", [])
stop_post_cmd = x2cmdlist(stop_post_cmd, False) + extra.pop(b"ExecStopPost", [])
start_pre_cmd = x2cmdlist(start_pre_cmd, False) + extra.pop(b"ExecStartPre", [])
start_post_cmd = x2cmdlist(start_post_cmd, False) + extra.pop(b"ExecStartPost", [])
if user_mode:
_wait_polling = _wait_polling or 0.5
with CExit() as ctexit, bus_factory() as bus, SDManager(bus=bus) as manager:
if pty:
if machine:
with pystemd.machine1.Machine(machine) as m:
pty_master, pty_path = m.Machine.OpenPTY()
else:
pty_master, pty_follower = ptylib.openpty()
pty_path = os.ttyname(pty_follower).encode()
ctexit.register(os.close, pty_master)
if slice_:
unit_properties[b"Slice"] = x2char_star(slice_)
if pty_path:
unit_properties.update(
{
b"StandardInput": b"tty",
b"StandardOutput": b"tty",
b"StandardError": b"tty",
b"TTYPath": pty_path,
}
)
if None not in (stdin, pty_master):
# lets set raw mode for stdin so we can forward input without
# waiting for a new line, but lets also make sure we return the
# attributes as they where after this method is done
stdin_attrs = tty.tcgetattr(stdin)
tty.setraw(stdin)
ctexit.register(tty.tcsetattr, stdin, tty.TCSAFLUSH, stdin_attrs)
selectors.append(stdin)
if None not in (stdout, pty_master):
if os.getenv("TERM"):
env[b"TERM"] = env.get(b"TERM", os.getenv("TERM").encode())
selectors.append(pty_master)
# lets be a friend and set the size of the pty.
winsize = fcntl.ioctl(
stdout, termios.TIOCGWINSZ, struct.pack("HHHH", 0, 0, 0, 0)
)
fcntl.ioctl(pty_master, termios.TIOCSWINSZ, winsize)
else:
unit_properties.update(
{
b"StandardInputFileDescriptor": get_fno(stdin) if stdin else stdin,
b"StandardOutputFileDescriptor": get_fno(stdout)
if stdout
else stdout,
b"StandardErrorFileDescriptor": get_fno(stderr)
if stderr
else stderr,
}
)
unit_properties.update(
{
b"Type": service_type,
b"Description": b"pystemd: " + name,
b"ExecStartPre": start_pre_cmd or None,
b"ExecStart": start_cmd,
b"ExecStartPost": start_post_cmd or None,
b"ExecStop": stop_cmd or None,
b"ExecStopPost": stop_post_cmd or None,
b"RemainAfterExit": remain_after_exit,
b"CollectMode": b"inactive-or-failed" if collect else None,
b"WorkingDirectory": cwd,
b"User": user,
b"Nice": nice,
b"RuntimeMaxUSec": runtime_max_usec,
b"Environment": [
b"%s=%s" % (x2char_star(key), x2char_star(value))
for key, value in env.items()
]
or None,
}
)
unit_properties.update(extra)
unit_properties = {k: v for k, v in unit_properties.items() if v is not None}
unit = Unit(name, bus=bus, _autoload=True)
if wait:
mstr = (
(
"type='signal',"
"sender='org.freedesktop.systemd1',"
"path='{}',"
"interface='org.freedesktop.DBus.Properties',"
"member='PropertiesChanged'"
)
.format(unit.path.decode())
.encode()
)
monbus = bus_factory()
monbus.open()
ctexit.register(monbus.close)
monitor = pystemd.DBus.Manager(bus=monbus, _autoload=True)
monitor.Monitoring.BecomeMonitor([mstr], 0)
monitor_fd = monbus.get_fd()
selectors.append(monitor_fd)
# start the process
unit_start_job = manager.Manager.StartTransientUnit(
name, b"fail", unit_properties
)
while wait:
_in, _, _ = select.select(selectors, [], [], _wait_polling)
if stdin in _in:
data = os.read(stdin, 1024)
os.write(pty_master, data)
if pty_master in _in:
try:
data = os.read(pty_master, 1024)
except OSError:
selectors.remove(pty_master)
else:
os.write(stdout, data)
if monitor_fd in _in:
m = monbus.process()
if m.is_empty():
continue
m.process_reply(False)
if (
m.get_path() == unit.path
and m.body[0] == b"org.freedesktop.systemd1.Unit"
):
_, message_job_path = m.body[1].get(b"Job", (0, b"/"))
if (
message_job_path != unit_start_job
and m.body[1].get(b"SubState") in EXIT_SUBSTATES
):
break
if _wait_polling and not _in and unit.Service.MainPID == 0:
# on usermode the subscribe to events does not work that well
# this is a temporary hack. you can always not wait on usermode.
break
if raise_on_fail:
if unit.Service.ExecMainStatus:
raise PystemdRunError(
"cmd {} exited with status {}".format(
cmd, unit.Service.ExecMainStatus
)
)
unit.load()
unit.bus_context = bus_factory
return unit
def main(**args):
# figure out explicit defines
defines = {}
for define in args['D']:
k, v, *_ = define.split('=', 2) + ['']
defines[k] = v
# and what class of TestCase to run
classes = []
if args.get('normal'):
classes.append(TestCase)
if args.get('reentrant'):
classes.append(ReentrantTestCase)
if args.get('valgrind'):
classes.append(ValgrindTestCase)
if not classes:
classes = [TestCase]
suites = []
for testpath in args['test_paths']:
# optionally specified test case/perm
testpath, *filter = testpath.split('#')
filter = [int(f) for f in filter]
# figure out the suite's toml file
if os.path.isdir(testpath):
testpath = testpath + '/*.toml'
elif os.path.isfile(testpath):
testpath = testpath
elif testpath.endswith('.toml'):
testpath = TEST_PATHS + '/' + testpath
else:
testpath = TEST_PATHS + '/' + testpath + '.toml'
# find tests
for path in glob.glob(testpath):
suites.append(TestSuite(path, classes, defines, filter, **args))
# sort for reproducability
suites = sorted(suites)
# generate permutations
for suite in suites:
suite.permute(**args)
# build tests in parallel
print('====== building ======')
makefiles = []
targets = []
for suite in suites:
makefile, target = suite.build(**args)
makefiles.append(makefile)
targets.append(target)
cmd = (['make', '-f', 'Makefile'] +
list(it.chain.from_iterable(['-f', m] for m in makefiles)) +
[target for target in targets])
mpty, spty = pty.openpty()
if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd, stdout=spty, stderr=spty)
os.close(spty)
mpty = os.fdopen(mpty, 'r', 1)
stdout = []
while True:
try:
line = mpty.readline()
except OSError as e:
if e.errno == errno.EIO:
break
raise
stdout.append(line)
if args.get('verbose'):
sys.stdout.write(line)
# intercept warnings
m = re.match(
'^{0}([^:]+):(\d+):(?:\d+:)?{0}{1}:{0}(.*)$'.format(
'(?:\033\[[\d;]*.| )*', 'warning'), line)
if m and not args.get('verbose'):
try:
with open(m.group(1)) as f:
lineno = int(m.group(2))
line = next(it.islice(f, lineno - 1, None)).strip('\n')
sys.stdout.write(
"\033[01m{path}:{lineno}:\033[01;35mwarning:\033[m "
"{message}\n{line}\n\n".format(path=m.group(1),
line=line,
lineno=lineno,
message=m.group(3)))
except:
pass
proc.wait()
if proc.returncode != 0:
if not args.get('verbose'):
for line in stdout:
sys.stdout.write(line)
sys.exit(-1)
print('built %d test suites, %d test cases, %d permutations' %
(len(suites), sum(len(suite.cases) for suite in suites),
sum(len(suite.perms) for suite in suites)))
total = 0
for suite in suites:
for perm in suite.perms:
total += perm.shouldtest(**args)
if total != sum(len(suite.perms) for suite in suites):
print('filtered down to %d permutations' % total)
# only requested to build?
if args.get('build'):
return 0
print('====== testing ======')
try:
for suite in suites:
suite.test(**args)
except TestFailure:
pass
print('====== results ======')
passed = 0
failed = 0
for suite in suites:
for perm in suite.perms:
if perm.result == PASS:
passed += 1
elif isinstance(perm.result, TestFailure):
sys.stdout.write(
"\033[01m{path}:{lineno}:\033[01;31mfailure:\033[m "
"{perm} failed\n".format(perm=perm,
path=perm.suite.path,
lineno=perm.lineno,
returncode=perm.result.returncode
or 0))
if perm.result.stdout:
if perm.result.assert_:
stdout = perm.result.stdout[:-1]
else:
stdout = perm.result.stdout
for line in stdout[-5:]:
sys.stdout.write(line)
if perm.result.assert_:
sys.stdout.write(
"\033[01m{path}:{lineno}:\033[01;31massert:\033[m "
"{message}\n{line}\n".format(**perm.result.assert_))
sys.stdout.write('\n')
failed += 1
if args.get('coverage'):
# collect coverage info
# why -j1? lcov doesn't work in parallel because of gcov limitations
cmd = (['make', '-j1', '-f', 'Makefile'] +
list(it.chain.from_iterable(['-f', m] for m in makefiles)) +
(['COVERAGETARGET=%s' % args['coverage']] if isinstance(
args['coverage'], str) else []) + [
suite.path + '.info'
for suite in suites if any(perm.result == PASS
for perm in suite.perms)
])
if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd,
stdout=sp.PIPE if not args.get('verbose') else None,
stderr=sp.STDOUT if not args.get('verbose') else None,
universal_newlines=True)
proc.wait()
if proc.returncode != 0:
if not args.get('verbose'):
for line in proc.stdout:
sys.stdout.write(line)
sys.exit(-1)
if args.get('gdb'):
failure = None
for suite in suites:
for perm in suite.perms:
if isinstance(perm.result, TestFailure):
failure = perm.result
if failure is not None:
print('======= gdb ======')
# drop into gdb
failure.case.test(failure=failure, **args)
sys.exit(0)
print('tests passed %d/%d (%.2f%%)' % (passed, total, 100 *
(passed / total if total else 1.0)))
print('tests failed %d/%d (%.2f%%)' % (failed, total, 100 *
(failed / total if total else 1.0)))
return 1 if failed > 0 else 0
def mk(name):
a, b = openpty()
masters.append(a)
slaves.append(b)
names.append(name)
def _run(self, cmd, in_data, sudoable=True, checkrc=True):
'''
Starts the command and communicates with it until it ends.
'''
display_cmd = list(map(shlex_quote, map(to_text, cmd)))
display.vvv(u'SSH: EXEC {0}'.format(u' '.join(display_cmd)), host=self.host)
# Start the given command. If we don't need to pipeline data, we can try
# to use a pseudo-tty (ssh will have been invoked with -tt). If we are
# pipelining data, or can't create a pty, we fall back to using plain
# old pipes.
p = None
if isinstance(cmd, (text_type, binary_type)):
cmd = to_bytes(cmd)
else:
cmd = list(map(to_bytes, cmd))
if not in_data:
try:
# Make sure stdin is a proper pty to avoid tcgetattr errors
master, slave = pty.openpty()
if PY3 and self._play_context.password:
p = subprocess.Popen(cmd, stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE, pass_fds=self.sshpass_pipe)
else:
p = subprocess.Popen(cmd, stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdin = os.fdopen(master, 'wb', 0)
os.close(slave)
except (OSError, IOError):
p = None
if not p:
if PY3 and self._play_context.password:
p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, pass_fds=self.sshpass_pipe)
else:
p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdin = p.stdin
# If we are using SSH password authentication, write the password into
# the pipe we opened in _build_command.
if self._play_context.password:
os.close(self.sshpass_pipe[0])
try:
os.write(self.sshpass_pipe[1], to_bytes(self._play_context.password) + b'\n')
except OSError as e:
# Ignore broken pipe errors if the sshpass process has exited.
if e.errno != errno.EPIPE or p.poll() is None:
raise
os.close(self.sshpass_pipe[1])
#
# SSH state machine
#
# Now we read and accumulate output from the running process until it
# exits. Depending on the circumstances, we may also need to write an
# escalation password and/or pipelined input to the process.
states = [
'awaiting_prompt', 'awaiting_escalation', 'ready_to_send', 'awaiting_exit'
]
# Are we requesting privilege escalation? Right now, we may be invoked
# to execute sftp/scp with sudoable=True, but we can request escalation
# only when using ssh. Otherwise we can send initial data straightaway.
state = states.index('ready_to_send')
if b'ssh' in cmd:
if self._play_context.prompt:
# We're requesting escalation with a password, so we have to
# wait for a password prompt.
state = states.index('awaiting_prompt')
display.debug(u'Initial state: %s: %s' % (states[state], self._play_context.prompt))
elif self._play_context.become and self._play_context.success_key:
# We're requesting escalation without a password, so we have to
# detect success/failure before sending any initial data.
state = states.index('awaiting_escalation')
display.debug(u'Initial state: %s: %s' % (states[state], self._play_context.success_key))
# We store accumulated stdout and stderr output from the process here,
# but strip any privilege escalation prompt/confirmation lines first.
# Output is accumulated into tmp_*, complete lines are extracted into
# an array, then checked and removed or copied to stdout or stderr. We
# set any flags based on examining the output in self._flags.
b_stdout = b_stderr = b''
b_tmp_stdout = b_tmp_stderr = b''
self._flags = dict(
become_prompt=False, become_success=False,
become_error=False, become_nopasswd_error=False
)
# select timeout should be longer than the connect timeout, otherwise
# they will race each other when we can't connect, and the connect
# timeout usually fails
timeout = 2 + self._play_context.timeout
rpipes = [p.stdout, p.stderr]
for fd in rpipes:
fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)
# If we can send initial data without waiting for anything, we do so
# before we call select.
if states[state] == 'ready_to_send' and in_data:
self._send_initial_data(stdin, in_data)
state += 1
while True:
rfd, wfd, efd = select.select(rpipes, [], [], timeout)
# We pay attention to timeouts only while negotiating a prompt.
if not rfd:
if state <= states.index('awaiting_escalation'):
# If the process has already exited, then it's not really a
# timeout; we'll let the normal error handling deal with it.
if p.poll() is not None:
break
self._terminate_process(p)
raise AnsibleError('Timeout (%ds) waiting for privilege escalation prompt: %s' % (timeout, to_native(b_stdout)))
# Read whatever output is available on stdout and stderr, and stop
# listening to the pipe if it's been closed.
if p.stdout in rfd:
b_chunk = p.stdout.read()
if b_chunk == b'':
rpipes.remove(p.stdout)
# When ssh has ControlMaster (+ControlPath/Persist) enabled, the
# first connection goes into the background and we never see EOF
# on stderr. If we see EOF on stdout, lower the select timeout
# to reduce the time wasted selecting on stderr if we observe
# that the process has not yet existed after this EOF. Otherwise
# we may spend a long timeout period waiting for an EOF that is
# not going to arrive until the persisted connection closes.
timeout = 1
b_tmp_stdout += b_chunk
display.debug("stdout chunk (state=%s):\n>>>%s<<<\n" % (state, to_text(b_chunk)))
if p.stderr in rfd:
b_chunk = p.stderr.read()
if b_chunk == b'':
rpipes.remove(p.stderr)
b_tmp_stderr += b_chunk
display.debug("stderr chunk (state=%s):\n>>>%s<<<\n" % (state, to_text(b_chunk)))
# We examine the output line-by-line until we have negotiated any
# privilege escalation prompt and subsequent success/error message.
# Afterwards, we can accumulate output without looking at it.
if state < states.index('ready_to_send'):
if b_tmp_stdout:
b_output, b_unprocessed = self._examine_output('stdout', states[state], b_tmp_stdout, sudoable)
b_stdout += b_output
b_tmp_stdout = b_unprocessed
if b_tmp_stderr:
b_output, b_unprocessed = self._examine_output('stderr', states[state], b_tmp_stderr, sudoable)
b_stderr += b_output
b_tmp_stderr = b_unprocessed
else:
b_stdout += b_tmp_stdout
b_stderr += b_tmp_stderr
b_tmp_stdout = b_tmp_stderr = b''
# If we see a privilege escalation prompt, we send the password.
# (If we're expecting a prompt but the escalation succeeds, we
# didn't need the password and can carry on regardless.)
if states[state] == 'awaiting_prompt':
if self._flags['become_prompt']:
display.debug('Sending become_pass in response to prompt')
stdin.write(to_bytes(self._play_context.become_pass) + b'\n')
self._flags['become_prompt'] = False
state += 1
elif self._flags['become_success']:
state += 1
# We've requested escalation (with or without a password), now we
# wait for an error message or a successful escalation.
if states[state] == 'awaiting_escalation':
if self._flags['become_success']:
display.debug('Escalation succeeded')
self._flags['become_success'] = False
state += 1
elif self._flags['become_error']:
display.debug('Escalation failed')
self._terminate_process(p)
self._flags['become_error'] = False
raise AnsibleError('Incorrect %s password' % self._play_context.become_method)
elif self._flags['become_nopasswd_error']:
display.debug('Escalation requires password')
self._terminate_process(p)
self._flags['become_nopasswd_error'] = False
raise AnsibleError('Missing %s password' % self._play_context.become_method)
elif self._flags['become_prompt']:
# This shouldn't happen, because we should see the "Sorry,
# try again" message first.
display.debug('Escalation prompt repeated')
self._terminate_process(p)
self._flags['become_prompt'] = False
raise AnsibleError('Incorrect %s password' % self._play_context.become_method)
# Once we're sure that the privilege escalation prompt, if any, has
# been dealt with, we can send any initial data and start waiting
# for output.
if states[state] == 'ready_to_send':
if in_data:
self._send_initial_data(stdin, in_data)
state += 1
# Now we're awaiting_exit: has the child process exited? If it has,
# and we've read all available output from it, we're done.
if p.poll() is not None:
if not rpipes or not rfd:
break
# We should not see further writes to the stdout/stderr file
# descriptors after the process has closed, set the select
# timeout to gather any last writes we may have missed.
timeout = 0
continue
# If the process has not yet exited, but we've already read EOF from
# its stdout and stderr (and thus removed both from rpipes), we can
# just wait for it to exit.
elif not rpipes:
p.wait()
break
# Otherwise there may still be outstanding data to read.
# close stdin after process is terminated and stdout/stderr are read
# completely (see also issue #848)
stdin.close()
if C.HOST_KEY_CHECKING:
if cmd[0] == b"sshpass" and p.returncode == 6:
raise AnsibleError('Using a SSH password instead of a key is not possible because Host Key checking is enabled and sshpass does not support this. Please add this host\'s fingerprint to your known_hosts file to manage this host.')
controlpersisterror = b'Bad configuration option: ControlPersist' in b_stderr or b'unknown configuration option: ControlPersist' in b_stderr
if p.returncode != 0 and controlpersisterror:
raise AnsibleError('using -c ssh on certain older ssh versions may not support ControlPersist, set ANSIBLE_SSH_ARGS="" (or ssh_args in [ssh_connection] section of the config file) before running again')
if p.returncode == 255 and in_data and checkrc:
raise AnsibleConnectionFailure('SSH Error: data could not be sent to the remote host. Make sure this host can be reached over ssh')
return (p.returncode, b_stdout, b_stderr)
def run(cmd,
print_error=True,
asynchronous=False,
stdin=False,
stderr=subprocess.STDOUT,
outfile=None,
env_vars=None,
inherit_cwd=False,
inherit_env=True,
tty=False):
env_dict = os.environ.copy() if inherit_env else {}
if env_vars:
env_dict.update(env_vars)
env_dict = dict([(k, to_str(str(v))) for k, v in env_dict.items()])
if tty:
asynchronous = True
stdin = True
try:
cwd = os.getcwd() if inherit_cwd else None
if not asynchronous:
if stdin:
return subprocess.check_output(cmd,
shell=True,
stderr=stderr,
env=env_dict,
stdin=subprocess.PIPE,
cwd=cwd)
output = subprocess.check_output(cmd,
shell=True,
stderr=stderr,
env=env_dict,
cwd=cwd)
return output.decode(config.DEFAULT_ENCODING)
stdin_arg = subprocess.PIPE if stdin else None
stdout_arg = open(outfile, 'ab') if isinstance(
outfile, six.string_types) else outfile
stderr_arg = stderr
if tty:
# Note: leave the "pty" import here (not supported in Windows)
import pty
master_fd, slave_fd = pty.openpty()
stdin_arg = slave_fd
stdout_arg = stderr_arg = None
# start the actual sub process
kwargs = {}
if is_linux() or is_mac_os():
kwargs['preexec_fn'] = os.setsid
process = subprocess.Popen(cmd,
shell=True,
stdin=stdin_arg,
bufsize=-1,
stderr=stderr_arg,
stdout=stdout_arg,
env=env_dict,
cwd=cwd,
**kwargs)
if tty:
# based on: https://stackoverflow.com/questions/41542960
def pipe_streams(*args):
while process.poll() is None:
r, w, e = select.select([sys.stdin, master_fd], [], [])
if sys.stdin in r:
d = os.read(sys.stdin.fileno(), 10240)
os.write(master_fd, d)
elif master_fd in r:
o = os.read(master_fd, 10240)
if o:
os.write(sys.stdout.fileno(), o)
FuncThread(pipe_streams).start()
return process
except subprocess.CalledProcessError as e:
if print_error:
print("ERROR: '%s': exit code %s; output: %s" %
(cmd, e.returncode, e.output))
sys.stdout.flush()
raise e
def check_output(self, command, *, print_on_silent_log=False):
"""Run a command and supply the output to callback functions"""
logger = logging.getLogger("Process")
res = []
mfd, sfd = pty.openpty()
flags = fcntl.fcntl(mfd, fcntl.F_GETFL)
fcntl.fcntl(mfd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
process = subprocess.Popen(command, stderr=sfd, stdout=sfd, bufsize=0)
logger.log(ProcessWrapper.loglevel, "[%d] command: %s", process.pid,
" ".join(command))
# do not register/unregister already registered print_callback
if ProcessWrapper.print_callback in self.callbacks:
print_on_silent_log = False
if print_on_silent_log and logger.getEffectiveLevel(
) > ProcessWrapper.loglevel:
self.enable_print()
# close sfd so we notice when the child is gone
os.close(sfd)
# get a file object from the fd
buf = b""
while True:
try:
raw = os.read(mfd, 4096)
except BlockingIOError as ex:
# wait for new data and retry
select.select([mfd], [], [mfd], 0.1)
continue
except OSError as e:
if e.errno == 5:
break
raise
if raw:
buf += raw
*parts, buf = buf.split(b'\r')
res.extend(parts)
for part in parts:
for callback in self.callbacks:
callback(part, process)
process.poll()
if process.returncode is not None:
break
os.close(mfd)
process.wait()
if buf:
# process incomplete line
res.append(buf)
if buf[-1] != b'\n':
buf += b'\n'
for callback in self.callbacks:
callback(buf, process)
if print_on_silent_log and logger.getEffectiveLevel(
) > ProcessWrapper.loglevel:
self.disable_print()
if process.returncode != 0:
raise subprocess.CalledProcessError(process.returncode,
command,
output=b'\r'.join(res))
# this converts '\r\n' to '\n' to be more compatible to the behaviour
# of the normal subprocess module
return b'\n'.join([r.strip(b'\n') for r in res])
def test(self,
exec=[],
persist=False,
cycles=None,
gdb=False,
failure=None,
disk=None,
**args):
# build command
cmd = exec + [
'./%s.test' % self.suite.path,
repr(self.caseno),
repr(self.permno)
]
# persist disk or keep in RAM for speed?
if persist:
if not disk:
disk = self.suite.path + '.disk'
if persist != 'noerase':
try:
with open(disk, 'w') as f:
f.truncate(0)
if args.get('verbose'):
print('truncate --size=0', disk)
except FileNotFoundError:
pass
cmd.append(disk)
# simulate power-loss after n cycles?
if cycles:
cmd.append(str(cycles))
# failed? drop into debugger?
if gdb and failure:
ncmd = ['gdb']
if gdb == 'assert':
ncmd.extend(['-ex', 'r'])
if failure.assert_:
ncmd.extend(['-ex', 'up 2'])
elif gdb == 'main':
ncmd.extend([
'-ex',
'b %s:%d' % (self.suite.path, self.code_lineno), '-ex', 'r'
])
ncmd.extend(['--args'] + cmd)
if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in ncmd))
signal.signal(signal.SIGINT, signal.SIG_IGN)
sys.exit(sp.call(ncmd))
# run test case!
mpty, spty = pty.openpty()
if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd, stdout=spty, stderr=spty)
os.close(spty)
mpty = os.fdopen(mpty, 'r', 1)
stdout = []
assert_ = None
try:
while True:
try:
line = mpty.readline()
except OSError as e:
if e.errno == errno.EIO:
break
raise
stdout.append(line)
if args.get('verbose'):
sys.stdout.write(line)
# intercept asserts
m = re.match(
'^{0}([^:]+):(\d+):(?:\d+:)?{0}{1}:{0}(.*)$'.format(
'(?:\033\[[\d;]*.| )*', 'assert'), line)
if m and assert_ is None:
try:
with open(m.group(1)) as f:
lineno = int(m.group(2))
line = (next(it.islice(f, lineno - 1,
None)).strip('\n'))
assert_ = {
'path': m.group(1),
'line': line,
'lineno': lineno,
'message': m.group(3)
}
except:
pass
except KeyboardInterrupt:
raise TestFailure(self, 1, stdout, None)
proc.wait()
# did we pass?
if proc.returncode != 0:
raise TestFailure(self, proc.returncode, stdout, assert_)
else:
return PASS