def __open_process(self, logger, cmd, env, cwd, **kwargs): if logger: logger.info('Running command: %r' % cmd) if env: logger.info('With env (showing only all caps):') for key, val in env.iteritems(): if re.match(r'[A-Z0-9_]+', key): logger.info(' %-20s = %s' % (key,val)) logger.info('With CWD: %s' % (cwd or os.getcwd())) logger.info('-' * 100) # provide tty to enable line buffering. master_out_fd, slave_out_fd = pty.openpty() master_err_fd, slave_err_fd = pty.openpty() process = subprocess.Popen( cmd, cwd=cwd, env=env, shell=False, bufsize=1, stderr=slave_err_fd, stdout=slave_out_fd, close_fds=True) return (process, (master_out_fd, slave_out_fd), (master_err_fd, slave_err_fd))
def mkpty(): master1, slave = pty.openpty() slaveName1 = os.ttyname(slave) master2, slave = pty.openpty() slaveName2 = os.ttyname(slave) print '\nslave device names: ', slaveName1, slaveName2 return master1, master2
def __init__(self, ip=get_ipython()): # For the registration self.shell = ip self.nbOutStream = sys.stdout self.nbErrStream = sys.stderr self.pyOutStream = sys.__stdout__ self.pyErrStream = sys.__stderr__ self.outStreamPipe_in = pty.openpty()[1] self.errStreamPipe_in = pty.openpty()[1] os.dup2(self.outStreamPipe_in, self.pyOutStream.fileno()) os.dup2(self.errStreamPipe_in, self.pyErrStream.fileno()) self.ioHandler = handlers.IOHandler() self.flag = True self.outString = "" self.errString = "" self.asyncCapturer = handlers.Runner(self.syncCapture) self.isFirstPreExecute = True self.isFirstPostExecute = True
def run(self): try: # use a pty. This enforces unbuffered output and thus # allows for fast update master_out_fd, slave_out_fd = pty.openpty() master_in_fd, self.slave_in_fd = pty.openpty() self.proc = subprocess.Popen(self.cmd, stdin=master_in_fd, stdout=slave_out_fd, stderr=slave_out_fd) except: self.finished.emit(False) return # listen to process' output while self.proc.poll() == None: try: if select.select([master_out_fd], [], [master_out_fd], .1)[0]: output = os.read(master_out_fd, 100) if output: self.output(str(output, "utf-8")) except InterruptedError: pass os.close(master_out_fd) os.close(slave_out_fd) os.close(master_in_fd) os.close(self.slave_in_fd) self.finished.emit(self.proc.wait() == 0)
def __init__(self, cmd, capturestderr=False, bufsize=-1): """ Popen3 class (isn't this actually Popen4, capturestderr = False?) that uses ptys instead of pipes, to allow inline reading (instead of potential i/o buffering) of output from the child process. It also stores the cmd its running (as a string) and the thread that created the object, for later use """ import pty # NOTE: most of this is cutnpaste from Popen3 minus the openpty calls # popen2._cleanup() self.prettyCmd = cmd cmd = self.parseCmdToList(cmd) self.cmd = cmd self.threadIdent = thread.get_ident() p2cread, p2cwrite = pty.openpty() c2pread, c2pwrite = pty.openpty() if capturestderr: errout, errin = pty.openpty() self.pid = os.fork() if self.pid == 0: # Child os.dup2(p2cread, 0) os.dup2(c2pwrite, 1) if capturestderr: os.dup2(errin, 2) self._run_child(cmd) os.close(p2cread) self.tochild = os.fdopen(p2cwrite, "w", bufsize) os.close(c2pwrite) self.fromchild = os.fdopen(c2pread, "r", bufsize) if capturestderr: os.close(errin) self.childerr = os.fdopen(errout, "r", bufsize) else: self.childerr = None
def __init__(self, args, cwd=None, env=None, stdin=False, echo_stdout=True, echo_stderr=True, capture_stdout=False, capture_stderr=False): if cwd is not None: cwd = str(cwd) logging.debug('Running {}...'.format(args[0])) logging.debug('Parameters: {}'.format(args)) logging.debug('Working directory: {}'.format(cwd)) logging.debug('Environment: {}'.format(env)) logging.debug('Echo: stdout: {}, stderr: {}'.format(echo_stdout, echo_stderr)) self.args = args self.buffer_stdout = bytearray() self.buffer_stderr = bytearray() master_stdout, slave_stdout = pty.openpty() master_stderr, slave_stderr = pty.openpty() Process.set_nonblocking(master_stdout) Process.set_nonblocking(master_stderr) self.process = subprocess.Popen(args, bufsize=0, cwd=cwd, env=env, stdin=stdin, stdout=slave_stdout, stderr=slave_stderr) pass_to_stdout = sys.stdout.buffer if echo_stdout else None pass_to_stderr = sys.stderr.buffer if echo_stderr else None self._reader_stdout = self.reader(master_stdout, capture_stdout, self.buffer_stdout, pass_to_stdout) self._reader_stderr = self.reader(master_stderr, capture_stderr, self.buffer_stderr, pass_to_stderr)
def tty_capture(cmd, bytes_input): """Capture the output of cmd with bytes_input to stdin, with stdin, stdout and stderr as TTYs.""" mo, so = pty.openpty() # provide tty to enable line-buffering me, se = pty.openpty() mi, si = pty.openpty() fdmap = {mo: 'stdout', me: 'stderr', mi: 'stdin'} p = subprocess.Popen( cmd, bufsize=1, stdin=si, stdout=so, stderr=se, close_fds=True) os.write(mi, bytes_input) timeout = .04 # seconds res = {'stdout': b'', 'stderr': b''} while True: ready, _, _ = select.select([mo, me], [], [], timeout) if ready: for fd in ready: data = os.read(fd, 512) if not data: break res[fdmap[fd]] += data elif p.poll() is not None: # select timed-out break # p exited for fd in [si, so, se, mi, mo, me]: os.close(fd) # can't do it sooner: it leads to errno.EIO error p.wait() return p.returncode, res['stdout'], res['stderr']
def _create_pty_or_pipe(copy_term_size=None): """ Try to create a pty and if then fails then create a normal pipe instead. @param copy_term_size: If a tty file descriptor is given then the term size will be copied to the pty. @type copy_term_size: int @rtype: tuple @returns: A tuple of (is_pty, master_fd, slave_fd) where is_pty is True if a pty was successfully allocated, and False if a normal pipe was allocated. """ got_pty = False global _disable_openpty, _fbsd_test_pty if _fbsd_test_pty and not _disable_openpty: # Test for python openpty breakage after freebsd7 to freebsd8 # upgrade, which results in a 'Function not implemented' error # and the process being killed. pid = os.fork() if pid == 0: pty.openpty() os._exit(os.EX_OK) pid, status = os.waitpid(pid, 0) if (status & 0xff) == 140: _disable_openpty = True _fbsd_test_pty = False if _disable_openpty: master_fd, slave_fd = os.pipe() else: try: master_fd, slave_fd = pty.openpty() got_pty = True except EnvironmentError as e: _disable_openpty = True writemsg("openpty failed: '%s'\n" % str(e), noiselevel=-1) del e master_fd, slave_fd = os.pipe() if got_pty: # Disable post-processing of output since otherwise weird # things like \n -> \r\n transformations may occur. mode = termios.tcgetattr(slave_fd) mode[1] &= ~termios.OPOST termios.tcsetattr(slave_fd, termios.TCSANOW, mode) if got_pty and \ copy_term_size is not None and \ os.isatty(copy_term_size): rows, columns = get_term_size() set_term_size(rows, columns, slave_fd) return (got_pty, master_fd, slave_fd)
def _execute_process_pty(cmd, cwd, env, shell, stderr_to_stdout=True): stdout_master, stdout_slave = None, None stderr_master, stderr_slave = None, None fds_to_close = [stdout_master, stdout_slave, stderr_master, stderr_slave] try: stdout_master, stdout_slave = pty.openpty() if stderr_to_stdout: stderr_master, stderr_slave = stdout_master, stdout_slave else: stderr_master, stderr_slave = pty.openpty() p = None while p is None: try: p = Popen( cmd, stdin=stdout_slave, stdout=stderr_slave, stderr=STDOUT, cwd=cwd, env=env, shell=shell, close_fds=False) except OSError as exc: # This can happen if a file you are trying to execute is being # written to simultaneously on Linux # (doesn't appear to happen on OS X) # It seems like the best strategy is to just try again later # Worst case is that the file eventually gets deleted, then a # different OSError would occur. if 'Text file busy' in '{0}'.format(exc): # This is a transient error, try again shortly time.sleep(0.01) continue raise # This causes the below select to exit when the subprocess closes. # On Linux, this sometimes causes Errno 5 OSError's when os.read # is called from within _yield_data, so on Linux _yield_data # catches and passes on that particular OSError. os.close(stdout_slave) if not stderr_to_stdout: os.close(stderr_slave) left_overs = {stdout_master: b'', stderr_master: b''} fds = [stdout_master] if stderr_master != stdout_master: fds.append(stderr_master) finally: # Make sure we don't leak file descriptors _close_fds(fds_to_close) # The linesep with pty's always seems to be "\r\n", even on OS X return _yield_data(p, fds, left_overs, "\r\n", fds_to_close)
def __init__(self, path, args, stdout_cb, stderr_cb, exited_cb, timeout_cb, timeout=15): threading.Thread.__init__(self) self.path = os.path.abspath(path) if HAS_SANDBOX_SUPPORT: self.sandboxpy = os.path.join(os.path.dirname(os.path.abspath(__file__)), "MiniSandbox.py") self.arguments = ["python", self.sandboxpy, os.path.dirname(self.path), self.path] else: self.sandboxpy = os.path.join(os.path.dirname(os.path.abspath(__file__)), "NoSandbox.py") self.arguments = ["python", self.sandboxpy, os.path.dirname(self.path), self.path] self.arguments.extend(args) self.master, slave = pty.openpty() self.proc = subprocess.Popen(self.arguments, stdin=slave, stdout=slave, stderr=subprocess.PIPE, close_fds=True) self.stdout_cb = stdout_cb self.stderr_cb = stderr_cb self.exited_cb = exited_cb self.timeout_cb = timeout_cb self.timeout = timeout self.input_queue = []
def start_subprocess(self): """Start octave using a subprocess (no tty support)""" errmsg = ('\n\nPlease install GNU Octave and put it in your path\n') ON_POSIX = 'posix' in sys.builtin_module_names if self.use_pty: master, slave = pty.openpty() self.wfid, self.rfid = master, master rpipe, wpipe = slave, slave else: self.rfid, wpipe = os.pipe() rpipe, self.wfid = os.pipe() kwargs = dict(close_fds=ON_POSIX, bufsize=0, stdin=rpipe, stderr=wpipe, stdout=wpipe) if os.name == 'nt': startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW kwargs['startupinfo'] = startupinfo try: proc = subprocess.Popen(['octave', '-q', '--braindead'], **kwargs) except OSError: # pragma: no cover raise Oct2PyError(errmsg) else: self.reader = _Reader(self.rfid, self.read_queue) return proc
def handle(self): masterFd, slaveFd = pty.openpty() try: # if we're not in the main thread, this will not work. signal.signal(signal.SIGTTOU, signal.SIG_IGN) except: pass pid = os.fork() if pid: os.close(masterFd) raise SocketConnected(slaveFd, pid) # make parent process the pty slave - the opposite of # pty.fork(). In this setup, the parent process continues # to act normally, while the child process performs the # logging. This makes it simple to kill the logging process # when we are done with it and restore the parent process to # normal, unlogged operation. else: os.close(slaveFd) try: protocol = TelnetServerProtocolHandler(self.request, masterFd) protocol.handle() finally: os.close(masterFd) os._exit(1)
def __t_pty_tracker(self, trackerclass, **kwargs): def __drain(masterf): while True: termdata = masterf.read(1024) if len(termdata) == 0: break # # - Allocate a pty # - Create a thread to drain off the master side; without # this, the slave side will block when trying to write. # - Connect the prog tracker to the slave side # - Set it running # (master, slave) = pty.openpty() slavef = os.fdopen(slave, "w") masterf = os.fdopen(master, "r") t = threading.Thread(target=__drain, args=(masterf,)) t.start() p = trackerclass(output_file=slavef, **kwargs) progress.test_progress_tracker(p, gofast=True) slavef.close() t.join() masterf.close()
def _has_sudo(self, result): _master, slave = pty.openpty() os.setsid() fcntl.ioctl(slave, termios.TIOCSCTTY, 0) out, err, exit = run_command(['sudo', '-l', '-U', self.user[USER_NAME], 'sudo']) if exit == 0: debug("User %s is allowed to run sudo" % self.user[USER_NAME]) # sudo allows a wide range of configurations, such as controlling # which binaries the user can execute with sudo. # For now, we will just check whether the user is allowed to run # any command with sudo. out, err, exit = run_command(['sudo', '-l', '-U', self.user[USER_NAME]]) for line in out.split('\n'): if line and re.search("(ALL)", line): result.value = 1 debug("User %s can run any command with sudo" % result.value) return debug("User %s can only run some commands with sudo" % self.user[USER_NAME]) else: debug("User %s is not allowed to run sudo" % self.user[USER_NAME])
def run(self): self.ioloop = IOLoop.instance() (master_fd, slave_fd) = pty.openpty() # make stdout, stderr non-blocking fcntl.fcntl(master_fd, fcntl.F_SETFL, fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) self.master_fd = master_fd self.master = os.fdopen(master_fd) # listen to stdout, stderr self.ioloop.add_handler(master_fd, self._handle_subprocess_stdout, self.ioloop.READ) slave = os.fdopen(slave_fd) self.kwargs["stdout"] = slave self.kwargs["stderr"] = slave self.kwargs["close_fds"] = True self.pipe = subprocess.Popen(*self.args, **self.kwargs) self.stdin = self.pipe.stdin # check for process exit self.wait_callback = PeriodicCallback(self._wait_for_end, 250) self.wait_callback.start()
def ssh_output(cmd): """ Runs an SSH command and returns the stdout/stderr. :param cmd: command to run :type cmd: str :rtype: (str, str) """ # ssh must run with stdin attached to a tty master, slave = pty.openpty() proc = subprocess.Popen(cmd, stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=os.setsid, close_fds=True, shell=True) os.close(slave) # wait for the ssh connection time.sleep(8) # kill the whole process group os.killpg(os.getpgid(proc.pid), 15) os.close(master) stdout, stderr = proc.communicate() print('SSH STDOUT: {}'.format(stdout.decode('utf-8'))) print('SSH STDERR: {}'.format(stderr.decode('utf-8'))) return stdout, stderr
def run_command(cmd, cwd=None): master, slave = pty.openpty() p = None while p is None: try: p = Popen(cmd, stdin=slave, stdout=slave, stderr=STDOUT, cwd=cwd) except OSError as exc: if "Text file busy" in str(exc): # This is a transient error, try again shortly time.sleep(0.01) continue raise if sys.platform.startswith("darwin"): os.close(slave) # This causes the below select to exit when the subprocess closes left_over = b"" # Read data until the process is finished while p.poll() is None: incomming = left_over rlist, wlist, xlist = select.select([master], [], [], 0.1) if rlist: incomming += os.read(master, 1024) lines = incomming.splitlines(True) # keepends=True data, left_over = process_incomming_lines(lines, left_over) if data is None: continue yield data # Done os.close(master) yield p.returncode
def __init__(self, cmd, logger): self.thread = threading.Thread(target=TestProcess._run, args=(weakref.proxy(self),)) self.pty_master, self.pty_slave = pty.openpty() self.logger = logger self.process = None self.traces = "" self.cmd = cmd
def call_and_peek_output(cmd, shell=False): import pty import subprocess master, slave = pty.openpty() print cmd p = subprocess.Popen(cmd, shell=shell, stdin=None, stdout=slave, close_fds=True) os.close(slave) line = "" while True: try: ch = os.read(master, 1) except OSError: # We get this exception when the spawn process closes all references to the # pty descriptor which we passed him to use for stdout # (typically when it and its childs exit) break line += ch if ch == '\n': yield line line = "" if line: yield line ret = p.wait() if ret: raise subprocess.CalledProcessError(ret, cmd)
def __enter__(self): # prepare standard file descriptors for raw manipulation self.was_blocking = os.get_blocking(0) os.set_blocking(0, False) try: self.terminal_attr_stdin = termios.tcgetattr(0) self.terminal_attr_stdout = termios.tcgetattr(1) self.terminal_attr_stderr = termios.tcgetattr(2) tty.setraw(0) tty.setraw(1) tty.setraw(2) except termios.error: # probably redirected self.terminal_attr_stdin = None # redirect standard file descriptors to new PTY master, slave = pty.openpty() os.set_blocking(master, False) self.real_stdin = os.dup(0) self.real_stdout = os.dup(1) self.real_stderr = os.dup(2) os.close(0) os.close(1) os.close(2) os.dup2(slave, 0) os.dup2(slave, 1) os.dup2(slave, 2) os.close(slave) self.terminal_pipe = master # start REPL in separate thread threading.Thread(target=repl, args=(self,), daemon=True).start() return self
def _get_pty_pair(self, encoding='ascii'): master_fd, slave_fd = pty.openpty() master = os.fdopen(master_fd, 'rb', 0) out = os.fdopen(slave_fd, 'wb', 0) master = io.TextIOWrapper(master) out = io.TextIOWrapper(out) return master, out
def test_signal_failure(monkeypatch): import os import pty import signal from pyrepl.unix_console import UnixConsole def failing_signal(a, b): raise ValueError def really_failing_signal(a, b): raise AssertionError mfd, sfd = pty.openpty() try: with sane_term(): c = UnixConsole(sfd, sfd) c.prepare() c.restore() monkeypatch.setattr(signal, 'signal', failing_signal) c.prepare() monkeypatch.setattr(signal, 'signal', really_failing_signal) c.restore() finally: os.close(mfd) os.close(sfd)
def _node_ssh_output(args): # ssh must run with stdin attached to a tty master, slave = pty.openpty() cmd = ('ssh-agent /bin/bash -c ' + '"ssh-add /host-home/.vagrant.d/insecure_private_key ' + '2> /dev/null && dcos node ssh --option StrictHostKeyChecking=no' + ' {}"').format(' '.join(args)) proc = subprocess.Popen(cmd, stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=os.setsid, close_fds=True, shell=True) os.close(slave) # wait for the ssh connection time.sleep(8) # kill the whole process group os.killpg(os.getpgid(proc.pid), 15) os.close(master) return proc.communicate()
def lock(self): """Use conserver to lock the machine.""" # find out current status of console debug.verbose('executing "console -i %s" to check state' % self.get_machine_name()) proc = subprocess.Popen(["console", "-i", self.get_machine_name()], stdout=subprocess.PIPE) line = proc.communicate()[0] assert(proc.returncode == 0) # check that nobody else has it open for writing myuser = getpass.getuser() parts = line.strip().split(':') conname, child, contype, details, users, state = parts[:6] if users: for userinfo in users.split(','): mode, username, host, port = userinfo.split('@')[:4] if 'w' in mode and username != myuser: raise MachineLockedError # Machine is not free # run a console in the background to 'hold' the lock and read output debug.verbose('starting "console %s"' % self.get_machine_name()) # run on a PTY to work around terminal mangling code in console (self.masterfd, slavefd) = pty.openpty() self.lockprocess = subprocess.Popen(["console", self.get_machine_name()], close_fds=True, stdout=slavefd, stdin=slavefd) os.close(slavefd) # XXX: open in binary mode with no buffering # otherwise select.select() may block when there is data in the buffer self.console_out = os.fdopen(self.masterfd, 'rb', 0)
def spawn(self, argv=None, term=None): if argv is None: if "SHELL" in os.environ: argv = [os.environ["SHELL"]] elif "PATH" in os.environ: # searching sh in the path. It can be unusual like /system/bin/sh on android for shell in ["bash", "sh", "ksh", "zsh", "csh", "ash"]: for path in os.environ["PATH"].split(":"): fullpath = os.path.join(path.strip(), shell) if os.path.isfile(fullpath): argv = [fullpath] break if argv: break if not argv: argv = ["/bin/sh"] if term is not None: os.environ["TERM"] = term master, slave = pty.openpty() self.slave = slave self.master = os.fdopen(master, "rb+wb", 0) # open file in an unbuffered mode flags = fcntl.fcntl(self.master, fcntl.F_GETFL) assert flags >= 0 flags = fcntl.fcntl(self.master, fcntl.F_SETFL, flags | os.O_NONBLOCK) assert flags >= 0 self.prog = subprocess.Popen( shell=False, args=argv, stdin=slave, stdout=slave, stderr=subprocess.STDOUT, preexec_fn=prepare )
def __init__(self, parent=None): """Initialise Linux pseudo terminal """ QThread.__init__(self, parent) self.master, self.slave = pty.openpty() self.ptyname = os.ttyname(self.slave) self.stop = False
def start_proc(self): preexec = noop stdin = stdout = stderr = None cqlshlog.info("Spawning %r subprocess with args: %r and env: %r" % (self.exe_path, self.args, self.env)) if self.realtty: masterfd, slavefd = pty.openpty() preexec = (lambda: set_controlling_pty(masterfd, slavefd)) self.proc = subprocess.Popen((self.exe_path,) + tuple(self.args), env=self.env, preexec_fn=preexec, stdin=stdin, stdout=stdout, stderr=stderr, close_fds=False) os.close(slavefd) self.childpty = masterfd self.send = self.send_tty self.read = self.read_tty else: stdin = stdout = subprocess.PIPE stderr = subprocess.STDOUT self.proc = subprocess.Popen((self.exe_path,) + tuple(self.args), env=self.env, stdin=stdin, stdout=stdout, stderr=stderr, bufsize=0, close_fds=False) self.send = self.send_pipe if self.tty: self.winpty = WinPty(self.proc.stdout) self.read = self.read_winpty else: self.read = self.read_pipe
def command(cmd, exception=PyscaleError, sudo=False, shell=False): # fix unicode stuff cmd = str(cmd) # parse args if sudo: # XXX: --session-command vs --command(-c) # session-command seems to be better but is only available on CentOS & Co. # cmd = "su -c '%s'" % cmd cmd = "sudo -n bash -c '%s'" % cmd if not shell: cmd = shlex.split(cmd) # execute slave = None if sudo: # give su a pty master, slave = pty.openpty() out, err = GPopen(cmd, stdin=slave, stdout=sbp.PIPE, stderr = sbp.PIPE, shell=shell).communicate() # handle errors if not out and err: if exception: raise exception(err) else: print err return out
def __init_streams(self, stdin, stdout, stderr, unbuffered): self.stdin = self.stdout = self.stderr = None if unbuffered: master, slave = pty.openpty() if stdin is PIPE: self._child_stdin, self._stdin = (slave, master) if unbuffered else os.pipe() self.stdin = os.fdopen(self._stdin, 'w') elif isinstance(stdin, int): self._child_stdin, self._stdin = stdin, -1 elif stdin is not None: self._child_stdin, self._stdin = stdin.fileno(), -1 else: self._child_stdin = self._stdin = -1 if stdout is PIPE: self._stdout, self._child_stdout = (master, slave) if unbuffered else os.pipe() self.stdout = os.fdopen(self._stdout, 'r') elif isinstance(stdout, int): self._stdout, self._child_stdout = -1, stdout elif stdout is not None: self._stdout, self._child_stdout = -1, stdout.fileno() else: self._stdout = self._child_stdout = -1 if stderr is PIPE: self._stderr, self._child_stderr = os.pipe() self.stderr = os.fdopen(self._stderr, 'r') elif isinstance(stderr, int): self._stderr, self._child_stderr = -1, stderr elif stderr is not None: self._stderr, self._child_stderr = -1, stderr.fileno() else: self._stderr = self._child_stderr = -1
def start(self, conf_name): self.conf = get_config(conf_name) filename = export_conf(self.conf) # Prepare server ## Prepare storage file f = open(os.path.join(data_folder,'storage.cfg'), 'w') # USELESS #f.write('add_path .') #f.write('add_path $USERDIR') #f.write('add_path $DATADIR') # END USELESS f.write('add_path $CURRENTDIR') f.close() # Server Command if self.conf['conf'].get('server', ''): server_bin = self.conf['conf']['server'] else: server_bin = "`which teeworlds-server`" self.command = 'cd %s && %s -f %s' % (data_folder, server_bin, filename) # Open pty master, slave = pty.openpty() # Launch server self.process = Popen(self.command, shell=True, stdin=PIPE, stdout=slave, stderr=slave, close_fds=True) # Init thread self.server = TeeWorldsServer(self, master) self.live_stats = LiveStats() # Start Server self.server.start() # Start live stats thread self.live_stats.start()
import json import logging import os import pty import re import shlex import subprocess import tempfile from fcntl import fcntl, F_GETFL, F_SETFL from tests.cook import util logger = logging.getLogger(__name__) # Manually create a TTY that we can use as the default STDIN _STDIN_TTY = pty.openpty()[1] def decode(b): """Decodes as UTF-8""" return b.decode('UTF-8') def encode(o): """Encodes with UTF-8""" return str(o).encode('UTF-8') def stdout(cp): """Returns the UTF-8 decoded and stripped stdout of the given CompletedProcess""" return decode(cp.stdout).strip()
def _make_popen_named_args(self, others=None): if others is None: (_, slave) = pty.openpty() others = {'stdin': slave, 'stdout': slave, 'stderr': slave} super(RunPty, self)._make_popen_named_args(others=others)
def __fork_ptys(self): ''' Fork the PTY The major difference from the python source is that we separate the stdout from stderr output. ''' stdout_parent_fd, stdout_child_fd = pty.openpty() if stdout_parent_fd < 0 or stdout_child_fd < 0: raise TerminalException('Failed to open a TTY for stdout') stderr_parent_fd, stderr_child_fd = pty.openpty() if stderr_parent_fd < 0 or stderr_child_fd < 0: raise TerminalException('Failed to open a TTY for stderr') pid = os.fork() if pid < pty.CHILD: raise TerminalException('Failed to fork') elif pid == pty.CHILD: # Child. # Close parent FDs os.close(stdout_parent_fd) os.close(stderr_parent_fd) salt.utils.reinit_crypto() # ----- Make STDOUT the controlling PTY ---------------------> child_name = os.ttyname(stdout_child_fd) # Disconnect from controlling tty. Harmless if not already # connected try: tty_fd = os.open('/dev/tty', os.O_RDWR | os.O_NOCTTY) if tty_fd >= 0: os.close(tty_fd) # which exception, shouldn't we catch explicitly .. ? except: # pylint: disable=W0702 # Already disconnected. This happens if running inside cron pass # New session! os.setsid() # Verify we are disconnected from controlling tty # by attempting to open it again. try: tty_fd = os.open('/dev/tty', os.O_RDWR | os.O_NOCTTY) if tty_fd >= 0: os.close(tty_fd) raise TerminalException( 'Failed to disconnect from controlling tty. It is ' 'still possible to open /dev/tty.') # which exception, shouldn't we catch explicitly .. ? except: # pylint: disable=W0702 # Good! We are disconnected from a controlling tty. pass # Verify we can open child pty. tty_fd = os.open(child_name, os.O_RDWR) if tty_fd < 0: raise TerminalException( 'Could not open child pty, {0}'.format(child_name)) else: os.close(tty_fd) # Verify we now have a controlling tty. if os.name != 'posix': # Only do this check in not BSD-like operating systems. BSD-like operating systems breaks at this point tty_fd = os.open('/dev/tty', os.O_WRONLY) if tty_fd < 0: raise TerminalException( 'Could not open controlling tty, /dev/tty') else: os.close(tty_fd) # <---- Make STDOUT the controlling PTY ---------------------- # ----- Duplicate Descriptors -------------------------------> os.dup2(stdout_child_fd, pty.STDIN_FILENO) os.dup2(stdout_child_fd, pty.STDOUT_FILENO) os.dup2(stderr_child_fd, pty.STDERR_FILENO) # <---- Duplicate Descriptors -------------------------------- else: # Parent. Close Child PTY's salt.utils.reinit_crypto() os.close(stdout_child_fd) os.close(stderr_child_fd) return pid, stdout_parent_fd, stderr_parent_fd
def inpty(ctx: Context, cmd: Union[str, List], env: Dict = None, log: ByteString = None): """ inpty polls colorful stdout of poetry to the terminal This function executes command and use select + pty to pull process stdout and stderr to file-object 'log' in non-blocking way, so it can simultaneously logging to stdout and close by itself after the process is done. Args: ctx (click.Context): a click Context object which is used to control the flow cmd (str, List): command that will be passed to a subprocess env (Dict): copy current env settings to pty pseudo one to ensure process log (ByteString): A disposable container to collect stdout which is polled from the subprocess Returns: returncode (int): Result status of the process Raise: click.Abort: Abort by click if encounter error(s) """ try: exec_env = {} exec_env.update(os.environ) # copy the current OS environment into the local environment if env is not None: exec_env.update(env) # create master & slave pipes to receive stdout and stderr from process master, slave = pty.openpty() # check cmd is a list, otherwise if try to split a string and check it can be well reformat if not isinstance(cmd, list) and isinstance(cmd, str): # First check it will the same, then keep processing, otherwise abort the process assert cmd == " ".join(split(cmd)), ctx.abort() cmd = split(cmd) p = Popen(cmd, shell=False, env=exec_env, stdout=slave, stderr=slave) # Loop while the process is executing while p.poll() is None: # Loop long as the selct mechanism indicates there is data to be read from the buffer while len(select([master], [], [], 0)[0]) == 1: # Read up to a 1 KB chunk of data buf = os.read(master, 1024) # Stream data to the stdout's fd of 0 os.write(0, buf) if log is not None: log.write(buf) except Exception: ctx.abort() else: return p.returncode finally: # cleanup os.close(master) os.close(slave)
def start_connection(play_context, variables, task_uuid): ''' Starts the persistent connection ''' candidate_paths = [C.ANSIBLE_CONNECTION_PATH or os.path.dirname(sys.argv[0])] candidate_paths.extend(os.environ.get('PATH', '').split(os.pathsep)) for dirname in candidate_paths: ansible_connection = os.path.join(dirname, 'ansible-connection') if os.path.isfile(ansible_connection): display.vvvv("Found ansible-connection at path {0}".format(ansible_connection)) break else: raise AnsibleError("Unable to find location of 'ansible-connection'. " "Please set or check the value of ANSIBLE_CONNECTION_PATH") env = os.environ.copy() env.update({ # HACK; most of these paths may change during the controller's lifetime # (eg, due to late dynamic role includes, multi-playbook execution), without a way # to invalidate/update, ansible-connection won't always see the same plugins the controller # can. 'ANSIBLE_BECOME_PLUGINS': become_loader.print_paths(), 'ANSIBLE_CLICONF_PLUGINS': cliconf_loader.print_paths(), 'ANSIBLE_COLLECTIONS_PATH': to_native(os.pathsep.join(AnsibleCollectionConfig.collection_paths)), 'ANSIBLE_CONNECTION_PLUGINS': connection_loader.print_paths(), 'ANSIBLE_HTTPAPI_PLUGINS': httpapi_loader.print_paths(), 'ANSIBLE_NETCONF_PLUGINS': netconf_loader.print_paths(), 'ANSIBLE_TERMINAL_PLUGINS': terminal_loader.print_paths(), }) python = sys.executable master, slave = pty.openpty() p = subprocess.Popen( [python, ansible_connection, to_text(os.getppid()), to_text(task_uuid)], stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env ) os.close(slave) # We need to set the pty into noncanonical mode. This ensures that we # can receive lines longer than 4095 characters (plus newline) without # truncating. old = termios.tcgetattr(master) new = termios.tcgetattr(master) new[3] = new[3] & ~termios.ICANON try: termios.tcsetattr(master, termios.TCSANOW, new) write_to_file_descriptor(master, variables) write_to_file_descriptor(master, play_context.serialize()) (stdout, stderr) = p.communicate() finally: termios.tcsetattr(master, termios.TCSANOW, old) os.close(master) if p.returncode == 0: result = json.loads(to_text(stdout, errors='surrogate_then_replace')) else: try: result = json.loads(to_text(stderr, errors='surrogate_then_replace')) except getattr(json.decoder, 'JSONDecodeError', ValueError): # JSONDecodeError only available on Python 3.5+ result = {'error': to_text(stderr, errors='surrogate_then_replace')} if 'messages' in result: for level, message in result['messages']: if level == 'log': display.display(message, log_only=True) elif level in ('debug', 'v', 'vv', 'vvv', 'vvvv', 'vvvvv', 'vvvvvv'): getattr(display, level)(message, host=play_context.remote_addr) else: if hasattr(display, level): getattr(display, level)(message) else: display.vvvv(message, host=play_context.remote_addr) if 'error' in result: if play_context.verbosity > 2: if result.get('exception'): msg = "The full traceback is:\n" + result['exception'] display.display(msg, color=C.COLOR_ERROR) raise AnsibleError(result['error']) return result['socket_path']
def execute(self, context, args, in_opt_format=None, out_opt_format=None): # This function is complex. There are two major variables. First, # are we on Unix or Windows? This is effectively determined by # pty_available, though I suppose some Unixes might not have ptys. # Second, out_opt_format tells us whether we want to stream the # output as lines (out_opt_format is None), or as unbuffered byte chunks # (determined by bytearray/chunked). There is also a special hack # x-filedescriptor/special where we pass along a file descriptor from # the subprocess; this is used in unicode.py to directly read the output. using_pty_out = pty_available and (out_opt_format not in ( None, 'x-unix-pipe-file-object/special')) using_pty_in = pty_available and (in_opt_format is None) and \ context.input_is_first and hasattr(context.input, 'connect') _logger.debug("using pty in: %s out: %s", using_pty_in, using_pty_out) # TODO - we need to rework things so that we allocate only one pty per pipeline. # In the very common case of exactly one command, this doesn't matter, but # allocating two ptys will probably bite us in odd ways if someone does create # a pipeline. Maybe have a context.request_pty() function? if using_pty_in or using_pty_out: # We create a pseudo-terminal to ensure that the subprocess is line-buffered. # Yes, this is gross, but as far as I know there is no other way to # control the buffering used by subprocesses. (master_fd, slave_fd) = pty.openpty() # Set the terminal to not do any processing; if you change this, you'll also # need to update unicode.py most likely. attrs = termios.tcgetattr(master_fd) attrs[1] = attrs[1] & (~termios.OPOST) termios.tcsetattr(master_fd, termios.TCSANOW, attrs) _logger.debug("allocated pty fds %d %d", master_fd, slave_fd) if using_pty_out: stdout_target = slave_fd else: stdout_target = subprocess.PIPE if context.input is None: stdin_target = None if using_pty_in: stdin_target = slave_fd elif in_opt_format == 'x-unix-pipe-file-object/special': stdin_target = iter(context.input).next() else: stdin_target = subprocess.PIPE _logger.debug("using stdin target: %r", stdin_target) context.attribs['master_fd'] = master_fd else: _logger.debug( "no pty available or non-chunked output, not allocating fds") (master_fd, slave_fd) = (None, None) stdout_target = subprocess.PIPE if context.input is None: stdin_target = None elif in_opt_format == 'x-unix-pipe-file-object/special': stdin_target = iter(context.input).next() else: stdin_target = subprocess.PIPE _logger.debug("using stdin target: %r", stdin_target) subproc_args = { 'bufsize': 0, 'stdin': stdin_target, 'stdout': stdout_target, 'stderr': subprocess.STDOUT, 'cwd': context.cwd } fs_encoding = sys.getfilesystemencoding() stdin_encoding = sys.stdin.encoding _logger.debug("recoding path to %r, args to %r", fs_encoding, stdin_encoding) # We need to encode_sentence arguments to the system encoding because subprocess.py won't do it for us. if fs_encoding is not None: args[0] = args[0].encode(fs_encoding) if stdin_encoding is not None: args[1:] = map(lambda x: x.encode(stdin_encoding), args[1:]) if is_windows(): subproc_args['universal_newlines'] = True startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW import win32con startupinfo.wShowWindow = win32con.SW_HIDE elif is_unix(): subproc_args['close_fds'] = True # Support freedesktop.org startup notification # http://standards.freedesktop.org/startup-notification-spec/startup-notification-latest.txt if context.gtk_event_time: env = dict(os.environ) env['DESKTOP_STARTUP_ID'] = 'hotwire%d_TIME%d' % ( os.getpid(), context.gtk_event_time, ) subproc_args['env'] = env def preexec(): os.setsid() if using_pty_out and hasattr(termios, 'TIOCSCTTY'): # Set our controlling TTY fcntl.ioctl(1, termios.TIOCSCTTY, '') # We ignore SIGHUP by default, because some broken programs expect that the lifetime # of subcommands is tied to an open window, instead of until when the toplevel # process exits. signal.signal(signal.SIGHUP, signal.SIG_IGN) subproc_args['preexec_fn'] = preexec else: assert (False) subproc = subprocess.Popen(args, **subproc_args) if not subproc.pid: if master_fd is not None: os.close(master_fd) if slave_fd is not None: os.close(slave_fd) raise ValueError('Failed to execute %s' % (args[0], )) context.attribs['pid'] = subproc.pid if using_pty_in or using_pty_out: os.close(slave_fd) context.status_notify('pid %d' % (context.attribs['pid'], )) if in_opt_format == 'x-unix-pipe-file-object/special': stdin_target.close() # If we were passed a file descriptor from another SysBuiltin, close it here; # it's now owned by the child. elif context.input: if using_pty_in: stdin_stream = BareFdStreamWriter(master_fd) else: stdin_stream = subproc.stdin # FIXME hack - need to rework input streaming if context.input_is_first and hasattr(context.input, 'connect'): context.attribs['input_connected'] = True context.input.connect(self.__on_input, stdin_stream) else: MiniThreadPool.getInstance().run(self.__inputwriter, args=(context.input, stdin_stream)) if using_pty_out: stdout_read = None stdout_fd = master_fd else: stdout_read = subproc.stdout stdout_fd = subproc.stdout.fileno() if out_opt_format is None: for line in SysBuiltin.__unbuffered_readlines(stdout_read): yield line elif out_opt_format == 'bytearray/chunked': try: for buf in SysBuiltin.__unbuffered_read_pipe( stream=stdout_read, fd=stdout_fd): yield buf except OSError, e: pass
# INPUT = device for multiplexed data # INPUT_BAUD = input baud rate # OUTPUTS = devices for de-multiplexed data ["/dev/ttyADDRESS0", /dev/ttyADDRESS1", ...] # output baud rates don't matter since they are virtual pty devices that work on any baud rate INPUT = "/dev/ttyTHS2" INPUT_BAUD = 57600 OUTPUTS = [ "/dev/ttyNC0", "/dev/gps0", "/dev/roboclaw0", "/dev/ttyNC1", "/dev/imu0" ] output_devices = [] input_ser = serial.Serial(INPUT, INPUT_BAUD) for output_symlink in OUTPUTS: master, slave = pty.openpty() master_name = os.ttyname(master) slave_name = os.ttyname(slave) if os.path.lexists(output_symlink): os.unlink(output_symlink) print("unlinked %s" % output_symlink) os.symlink(slave_name, output_symlink) print("created symlink %s -> %s" % (slave_name, output_symlink)) output_devices.append({ "master": master, "slave": slave, "symlink": output_symlink, })
def measure(evl, argl, equations, evnames): warned = False all_events = gen_events(evl) ## use a pty because perf doesn't do enough fflush() otherwise outp, inp = pty.openpty() logfile = "ulog.%d" % (os.getpid()) run = PerfRun() run.execute([perf, "stat", "--log-fd", "%d" % (inp), "-e", all_events] + argl, logfile, evl) prev_timestamp = None evp = defaultdict(list) res = defaultdict(list) socket = "" try: if args.mock: f = open(logfile, 'r') else: f = os.fdopen(outp, 'r') os.close(inp) while True: try: # force line-by-line buffering l = f.readline() if not l: break except KeyboardInterrupt, exceptions.IOError: break l = l.strip() dbg("perf", l) if l.startswith('#') or l == "": continue if per_socket: ts, socket, _, rest = l.split(",", 3) l = ts + "," + rest # uncore// contains commas! m = re.match(r"([0-9.]+),([0-9]+|<.*>),(.*)$", l) if not m: print "PERF-UNREADABLE", l, continue timestamp = m.group(1) if timestamp != prev_timestamp: if per_socket and not args.quiet: warned = check_per_socket(res.keys(), warned) if evp: num = len(res) gen_res(evl * num, concat(res), concat(evp), equations * num, gennames(evnames, res.keys()), timestamp) res = defaultdict(list) evp = defaultdict(list) prev_timestamp = timestamp r = m.group(2) if r.startswith("<"): if r in perf_errors: r = perf_errors[r] else: r = "#NA" res[socket].append(r) evp[socket].append(m.group(3)) f.close() if args.mock: os.remove(logfile)
def run( cmd: Union[str, List[str]], print_error=True, asynchronous=False, stdin=False, stderr=subprocess.STDOUT, outfile=None, env_vars: Optional[Dict[AnyStr, AnyStr]] = None, inherit_cwd=False, inherit_env=True, tty=False, shell=True, cwd: str = None, ) -> Union[str, subprocess.Popen]: LOG.debug("Executing command: %s", cmd) env_dict = os.environ.copy() if inherit_env else {} if env_vars: env_dict.update(env_vars) env_dict = {k: to_str(str(v)) for k, v in env_dict.items()} if isinstance(cmd, list): # See docs of subprocess.Popen(...): # "On POSIX with shell=True, the shell defaults to /bin/sh. If args is a string, # the string specifies the command to execute through the shell. [...] If args is # a sequence, the first item specifies the command string, and any additional # items will be treated as additional arguments to the shell itself." # Hence, we should *disable* shell mode here to be on the safe side, to prevent # arguments in the cmd list from leaking into arguments to the shell itself. This will # effectively allow us to call run(..) with both - str and list - as cmd argument, although # over time we should move from "cmd: Union[str, List[str]]" to "cmd: List[str]" only. shell = False if tty: asynchronous = True stdin = True try: if inherit_cwd and not cwd: cwd = os.getcwd() if not asynchronous: if stdin: return subprocess.check_output(cmd, shell=shell, stderr=stderr, env=env_dict, stdin=subprocess.PIPE, cwd=cwd) output = subprocess.check_output(cmd, shell=shell, stderr=stderr, env=env_dict, cwd=cwd) return output.decode(config.DEFAULT_ENCODING) stdin_arg = subprocess.PIPE if stdin else None stdout_arg = open(outfile, "ab") if isinstance(outfile, str) else outfile stderr_arg = stderr if tty: # Note: leave the "pty" import here (not supported in Windows) import pty master_fd, slave_fd = pty.openpty() stdin_arg = slave_fd stdout_arg = stderr_arg = None # start the actual sub process kwargs = {} if is_linux() or is_mac_os(): kwargs["start_new_session"] = True process = subprocess.Popen( cmd, shell=shell, stdin=stdin_arg, bufsize=-1, stderr=stderr_arg, stdout=stdout_arg, env=env_dict, cwd=cwd, **kwargs, ) if tty: # based on: https://stackoverflow.com/questions/41542960 def pipe_streams(*args): while process.poll() is None: r, w, e = select.select([sys.stdin, master_fd], [], []) if sys.stdin in r: d = os.read(sys.stdin.fileno(), 10240) os.write(master_fd, d) elif master_fd in r: o = os.read(master_fd, 10240) if o: os.write(sys.stdout.fileno(), o) FuncThread(pipe_streams).start() return process except subprocess.CalledProcessError as e: if print_error: print("ERROR: '%s': exit code %s; output: %s" % (cmd, e.returncode, e.output)) sys.stdout.flush() raise e
parser = argparse.ArgumentParser(description="LiteX JTAG UART bridge tool") parser.add_argument("--config", default="openocd_xc7_ft2232.cfg", help="OpenOCD config file") parser.add_argument("--telnet-port", default="20000", help="OpenOCD telnet port") args = parser.parse_args() def openocd_jtag_telnet(): prog = OpenOCD(args.config) prog.stream(int(args.telnet_port)) m, s = pty.openpty() print("LiteX JTAG UART created: {}".format(os.ttyname(s))) openocd_jtag_telnet_thread = threading.Thread(target=openocd_jtag_telnet) openocd_jtag_telnet_thread.start() time.sleep(1) t = telnetlib.Telnet("localhost", int(args.telnet_port)) def pty2telnet(m): while True: r = os.read(m, 1) t.write(r) if r == bytes("\n".encode("utf-8")):
def mkpty(): master, slave = pty.openpty() slaveName = os.ttyname(slave) print '\nslave device names: ', slaveName return master, slaveName
def _start_connection(self, variables): ''' Starts the persistent connection ''' master, slave = pty.openpty() python = sys.executable def find_file_in_path(filename): # Check $PATH first, followed by same directory as sys.argv[0] paths = os.environ['PATH'].split(os.pathsep) + [os.path.dirname(sys.argv[0])] for dirname in paths: fullpath = os.path.join(dirname, filename) if os.path.isfile(fullpath): return fullpath raise AnsibleError("Unable to find location of '%s'" % filename) p = subprocess.Popen( [python, find_file_in_path('ansible-connection'), to_text(os.getppid())], stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) stdin = os.fdopen(master, 'wb', 0) os.close(slave) # Need to force a protocol that is compatible with both py2 and py3. # That would be protocol=2 or less. # Also need to force a protocol that excludes certain control chars as # stdin in this case is a pty and control chars will cause problems. # that means only protocol=0 will work. src = cPickle.dumps(self._play_context.serialize(), protocol=0) stdin.write(src) stdin.write(b'\n#END_INIT#\n') src = cPickle.dumps(variables, protocol=0) stdin.write(src) stdin.write(b'\n#END_VARS#\n') stdin.flush() (stdout, stderr) = p.communicate() stdin.close() if p.returncode == 0: result = json.loads(to_text(stdout, errors='surrogate_then_replace')) else: try: result = json.loads(to_text(stderr, errors='surrogate_then_replace')) except getattr(json.decoder, 'JSONDecodeError', ValueError): # JSONDecodeError only available on Python 3.5+ result = {'error': to_text(stderr, errors='surrogate_then_replace')} if 'messages' in result: for msg in result.get('messages'): display.vvvv('%s' % msg, host=self._play_context.remote_addr) if 'error' in result: if self._play_context.verbosity > 2: if result.get('exception'): msg = "The full traceback is:\n" + result['exception'] display.display(msg, color=C.COLOR_ERROR) raise AnsibleError(result['error']) return result['socket_path']
def __init__(self, testload, speed=4800, databits=8, parity='N', stopbits=1, progress=None): super(FakePTY, self).__init__(testload, progress) # Allow Serial: header to be overridden by explicit speed. if self.testload.serial: (speed, databits, parity, stopbits) = self.testload.serial self.speed = speed baudrates = { 0: termios.B0, 50: termios.B50, 75: termios.B75, 110: termios.B110, 134: termios.B134, 150: termios.B150, 200: termios.B200, 300: termios.B300, 600: termios.B600, 1200: termios.B1200, 1800: termios.B1800, 2400: termios.B2400, 4800: termios.B4800, 9600: termios.B9600, 19200: termios.B19200, 38400: termios.B38400, 57600: termios.B57600, 115200: termios.B115200, 230400: termios.B230400, } (self.fd, self.slave_fd) = pty.openpty() self.byname = os.ttyname(self.slave_fd) os.chmod( self.byname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH) (iflag, oflag, cflag, lflag, ispeed, ospeed, cc) = termios.tcgetattr(self.slave_fd) cc[termios.VMIN] = 1 cflag &= ~(termios.PARENB | termios.PARODD | termios.CRTSCTS) cflag |= termios.CREAD | termios.CLOCAL iflag = oflag = lflag = 0 iflag &= ~(termios.PARMRK | termios.INPCK) cflag &= ~(termios.CSIZE | termios.CSTOPB | termios.PARENB | termios.PARODD) if databits == 7: cflag |= termios.CS7 else: cflag |= termios.CS8 if stopbits == 2: cflag |= termios.CSTOPB # Warning: attempting to set parity makes Fedora lose its cookies if parity == 'E': iflag |= termios.INPCK cflag |= termios.PARENB elif parity == 'O': iflag |= termios.INPCK cflag |= termios.PARENB | termios.PARODD ispeed = ospeed = baudrates[speed] try: termios.tcsetattr(self.slave_fd, termios.TCSANOW, [iflag, oflag, cflag, lflag, ispeed, ospeed, cc]) except termios.error: raise TestLoadError("error attempting to set serial mode to %s " " %s%s%s" % (speed, databits, parity, stopbits))
def main(): args = docopt(__doc__, version=__version__) if args.pop('--debug-trace', False): args['--debug'] = "NOTSET" log.basicConfig(level=getattr(log, args['--debug'], log.INFO), format='%(levelname)s: %(message)s') log.debug(args) # Get compressed sizes zips = {} for fn in args['<zipfiles>']: info = subprocess.check_output(["7z", "l", fn]).strip() finfo = RE_SCN.findall(info) # builtin test: last line should be total sizes log.debug(finfo) totals = map(int, finfo[-1][:2]) # log.debug(totals) for s in range(2): assert (sum(map(int, (inf[s] for inf in finfo[:-1]))) == totals[s]) fcomp = dict((n, int(c if args['--compressed'] else u)) for (u, c, n) in finfo[:-1]) # log.debug(fcomp) # zips : {'zipname' : {'filename' : int(size)}} zips[fn] = fcomp # Extract cmd7zx = ["7z", "x", "-bd"] if args['--yes']: cmd7zx += ["-y"] log.info("Extracting from {:d} file(s)".format(len(zips))) with tqdm(total=sum(sum(fcomp.values()) for fcomp in zips.values()), unit="B", unit_scale=True) as tall: for fn, fcomp in zips.items(): md, sd = pty.openpty() ex = subprocess.Popen( cmd7zx + [fn], bufsize=1, stdout=md, # subprocess.PIPE, stderr=subprocess.STDOUT) os.close(sd) with io.open(md, mode="rU", buffering=1) as m: with tqdm(total=sum(fcomp.values()), disable=len(zips) < 2, leave=False, unit="B", unit_scale=True) as t: while True: try: l_raw = m.readline() except IOError: break l = l_raw.strip() if l.startswith("Extracting"): exname = l.lstrip("Extracting").lstrip() s = fcomp.get(exname, 0) # 0 is likely folders t.update(s) tall.update(s) elif l: if not any( l.startswith(i) for i in ("7-Zip ", "p7zip Version ", "Everything is Ok", "Folders: ", "Files: ", "Size: ", "Compressed: ")): if l.startswith("Processing archive: "): if not args['--silent']: t.write( t.format_interval(t.start_t - tall.start_t) + ' ' + l.lstrip("Processing archive: ")) else: t.write(l) ex.wait()
def main(): args = argopt(__doc__, version=__version__).parse_args() if args.debug_trace: args.debug = "NOTSET" logging.basicConfig(level=getattr(logging, args.debug, logging.INFO), format='%(levelname)s:%(message)s') log = logging.getLogger(__name__) log.debug(args) # Get compressed sizes zips = {} for fn in args.zipfiles: info = subprocess.check_output(["7z", "l", fn]).strip() # nosec finfo = RE_SCN.findall(info) # size|compressed|name # builtin test: last line should be total sizes log.debug(finfo) totals = map(int, finfo[-1][:2]) # log.debug(totals) for s in range(2): # size|compressed totals totals_s = sum(map(int, (inf[s] for inf in finfo[:-1]))) if totals_s != totals[s]: log.warn("%s: individual total %d != 7z total %d", fn, totals_s, totals[s]) fcomp = {n: int(c if args.compressed else u) for (u, c, n) in finfo[:-1]} # log.debug(fcomp) # zips : {'zipname' : {'filename' : int(size)}} zips[fn] = fcomp # Extract cmd7zx = ["7z", "x", "-bd"] if args.yes: cmd7zx += ["-y"] log.info("Extracting from %d file(s)", len(zips)) with tqdm(total=sum(sum(fcomp.values()) for fcomp in zips.values()), unit="B", unit_scale=True) as tall: for fn, fcomp in zips.items(): md, sd = pty.openpty() ex = subprocess.Popen( cmd7zx + [fn], bufsize=1, stdout=md, # subprocess.PIPE, stderr=subprocess.STDOUT) # nosec os.close(sd) with io.open(md, mode="rU", buffering=1) as m: with tqdm(total=sum(fcomp.values()), disable=len(zips) < 2, leave=False, unit="B", unit_scale=True) as t: if not hasattr(t, "start_t"): # disabled t.start_t = tall._time() while True: try: l_raw = m.readline() except IOError: break ln = l_raw.strip() if ln.startswith("Extracting"): exname = ln.lstrip("Extracting").lstrip() s = fcomp.get(exname, 0) # 0 is likely folders t.update(s) tall.update(s) elif ln: if not any( ln.startswith(i) for i in ("7-Zip ", "p7zip Version ", "Everything is Ok", "Folders: ", "Files: ", "Size: ", "Compressed: ")): if ln.startswith("Processing archive: "): if not args.silent: t.write(t.format_interval( t.start_t - tall.start_t) + ' ' + ln.replace("Processing archive: ", "")) else: t.write(ln) ex.wait()
def make_slave_pty(): master_pty, slave_pty = pty.openpty() yield slave_pty os.close(slave_pty) os.close(master_pty)
def run(self): #Opening pseudo terminal mfd, sfd = pty.openpty() #Measure system before run extend_psutil_measure = __psutil_measure__() self.logger.debug( ("MULTILINE:\n{1} Running" + " process:\n {0}\n {1}").format( self.cmd, __format_span_next_line__("*_*", self.config))) #start process popen_obj = Popen(self.cmd, stdin=sfd, stdout=sfd, stderr=sfd, shell=True) self.logger.info( ("\n{span:-^80}\n" + "{message: ^80}\n{span:-^80}\n").format( span="PROCESS", message=self.cmd)) #calculate stoptimes general_time_out = self.__stoptime__(self.general_timeout) ready_time_out = self.__stoptime__(self.ready_timeout) response_time_out = self.__stoptime__(self.response_timeout) #<Play wiht pids:> self.logger.debug("Python process pid is: %i" % os.getpid()) #Get real pid of process by sh pid # - becouse we ran this process in shell mode # and we need to find all children of this process (must be only one of cource). real_pid = self.get_children_of_pid(popen_obj.pid) process_dubbed = False if real_pid != popen_obj.pid: process_dubbed = True self.pids = [real_pid, popen_obj.pid] self.logger.info("Shell /bin/sh pid: %i" % popen_obj.pid) self.logger.info("Sub process pid is: %i" % real_pid) else: self.pids = [real_pid] self.logger.info("Sub process pid is: %i" % real_pid) #</:Play wiht pids> #preparing result object self.result = self.__result__( real_pid, self.logger, extend_psutil_measure=extend_psutil_measure) communicator = None jogging_status = [".", "..", "...", ".."] jogging_status_idx = 0 jogging = False while True: self.logger.debug( ("MULTILINE:\n{1}\nCurrent time is:" + " {4}\n{1}Times for out:\n\t\t -" + " response_time_out = {0}\n\t\t -" + " general_time_out={2}\n\t\t - " + "ready_time_out={3}\n {1}").format( response_time_out, __format_span_next_line__("*_*", self.config), general_time_out, ready_time_out, float(time.time()))) #<Select result from pty by file description> self.logger.debug("Before select") #Calculate time that we spent for select response time_spend_for_respons = float(time.time()) r, w, e = select([mfd], [], [mfd], self.select_timeout) time_spend_for_respons = float( time.time()) - time_spend_for_respons self.logger.debug("After select") #</Select result from pty by file description> #Update status of process popen_obj.poll() #<system measure of current process> #Update measure self.result.measure.update() if hasattr(self.result.measure, "cpu_usage_average"): self.logger.debug("Current cpu usage average is : %f" % (self.result.measure.cpu_usage_average)) #</system measure of current process> self.result.return_code = popen_obj.returncode #recive cmd`s from queue in thread mode if (self.thread_mode): if not self.queue.empty(): queue_command = self.queue.get() if "KILL" in queue_command.upper(): self.logger.debug("KILL signal recived") self.stop() self.queue.task_done() elif ("jogging" in queue_command): jogging = True if jogging: if jogging_status_idx < (len(jogging_status) - 1): jogging_status_idx += 1 else: jogging_status_idx = 0 self.logger.info( "Pid %s work in thread%s" % (real_pid, jogging_status[jogging_status_idx])) communicator_decoded = "" if mfd in r: #self.logger.debug("Return code: %s " % str(self.result.return_code)) #Reading data from pty communicator = os.read(mfd, 10240) #communicator = communicator.replace(b'\x08', b'') ???????????? #Sometimes event problem with decoding of unrecognize simbols # because screen out from process came in two parts # and simbols starting at the end of one part ended at the begin of other - # - in two differents communicators variable (divided in time) # todo: need first collect than decode try: communicator_decoded = communicator.decode( self.config.common.default_codepage) # self.logger.debug("Decoded std(err)out: %s" % (repr(communicator_decoded))) except UnicodeDecodeError as ude: self.logger.debug( "Durring decoding result to %s raised error: %s\t-\t Try to workaroud." % (self.config.common.default_codepage, ude)) start_pos = ude.args[2] end_pos = ude.args[3] bad_bit = communicator[start_pos:end_pos] self.logger.debug( "Bad simbol is %s, between %i and %i posions.\t Will be trim part: %s" % (bad_bit, start_pos, end_pos, communicator[start_pos:])) communicator_trim = communicator[:start_pos] communicator_decoded = communicator_trim.decode( self.config.common.default_codepage) #<Sudo mechanism:> # !ATTENTION!: use -S option with sudo for recive password through stdin if "password for" in communicator_decoded: self.logger.debug("Asking for root password: %s" % communicator_decoded) if hasattr(self.config.common, "user_password"): pswd = bytes( "%s\n" % (self.config.common.user_password), self.config.common.default_codepage) if "Sorry, try again" in communicator_decoded: self.logger.warning( "!!ERROR: Password for sudo user %s is uncorrect %s " % (self.config.common.user_name, self.config.common.user_password)) self.result.errors_out_decoded = communicator_decoded self.logger.debug("Password set to %s" % (repr(pswd))) os.write(mfd, pswd) continue else: self.logger.info( "Sudo password is not provide! Stop process and exit..." ) self.result.errors_out_decoded = "Sudo password is not provide!" if (self.thread_mode): self.stop() #</Sudo mechanism> #Check if process returned more than 0 meaning error if (self.result.return_code is not None) and (self.result.return_code > 0): self.logger.warning("!!ERROR: Returned non zero! %s " % (communicator_decoded)) self.result.errors_out_decoded = communicator_decoded self.result.errors_out_bytes = communicator #else work with result else: self.logger.debug("\t++: %s " % communicator_decoded) response_time_out = self.__stoptime__( self.response_timeout) self.result.std_out_decoded += communicator_decoded self.result.std_out_bytes += communicator #<parce output by regular exprasion> if (len(communicator_decoded) > 0) and (self.regexp_pars is not None): self.logger.debug( "regexp_pars is not none. We are starting parsing standart output to class field..." ) for key, value in self.regexp_pars.items(): try: #self.logger.debug("Commu decode repr: %s" % (repr(communicator_decoded)) ) re_search = re.search(value, communicator_decoded, re.MULTILINE) if re_search is not None: setattr(self.result.screen_capture, key, re_search.group(key)) self.logger.debug( "Great we found %s it is %s" % (key, re_search.group(key))) else: self.logger.debug( "!!WARNING: Couldn't find %s" % key) except IndexError: self.logger.debug( "!!WARNING: Output hasn't this group of data" ) #</parce output by regular exprasion> #Word after that we will be ready to communicate or just stop(kill or something like this) this process if (self.wait_word) and (self.wait_word in communicator_decoded) and ( ready_time_out < time.time()): self.logger.debug("Catch wait word!") # - thread mode if (self.thread_mode): if (not jogging): #<work with thread> self.logger.debug( "\t-thread mode case starting preparing communicate..." ) #queue_get = self.queue.get() self.logger.debug( "\t-thread: %s, recived %s" % (str(self.id), queue_command[::-1])) self.queue.task_done() #</work with thread> # - single mode else: command = shlex.split( self.stop_cmd.format( pids=" ".join( [str(pi) for pi in self.pids]), result=self.result #port=self.result.screen_capture.esrv_port if hasattr(self.result.screen_capture, 'esrv_port'} else "" )) self.logger.debug( "\t\t-single mode case starting stop procedure by the command: %s" % (" ".join(command))) killer_return = run(command, stdout=PIPE, stderr=PIPE) if killer_return.returncode > 0: #softly exit not immplimented raise Exception( "Process stop errors! Return more than zerro. Error: %s " % (self.result.stderr)) #<Timeout predicates> #General time out exit if (general_time_out < time.time()): self.result.errors_out_decoded += "General timeout pass out! pass %f sec" % ( general_time_out) self.logger.warning(self.result.errors_out_decoded) if (self.thread_mode): if (self.thread_mode): try: self.queue.task_done() except: pass self.stop() self.result.return_code = 1 break #Response time out exit if (response_time_out < time.time()): self.result.errors_out_decoded += "Wait response timeout pass out! pass %f sec" % ( response_time_out) self.logger.warning(self.result.errors_out_decoded) if (self.thread_mode): if (self.thread_mode): try: self.queue.task_done() except: pass self.stop() self.result.return_code = 1 break #</Timeout predicates> self.logger.debug("self.result.return_code: '%s' " % (str(self.result.return_code))) #<error predicat> if (self.result.return_code is not None) and (self.result.return_code > 0): self.logger.debug( "A-a-a-a! its returned more than zerro!! Stopping...") if (len(self.result.errors_out_decoded) < 1): if (communicator_decoded in locals().keys() and communicator_decoded is not None): error_descr = "Unrecognized error:\n %s" % ( communicator_decoded) else: error_descr = "Unrecognized error." self.result.errors_out_decoded = error_descr if (self.thread_mode): try: self.queue.task_done() except: pass self.stop() break #</error predicat> #if return zerro or below zerro meaning that we successfully done elif (self.result.return_code is not None): if (self.thread_mode): self.queue.task_done() break self.logger.debug("{1} Finish prompt line {0} ...\n {1}".format( self.cmd, __format_span_next_line__("*_*", self.config))) return self.result
def call_command(command, command_timeout=None, output_timeout=None): """Call the given command with optional timeouts. This function calls the given command in a shell environment and applies a timeout on the whole command as well as a timeout on the stdout/stderr streams. The function returns the tuple (return code, stdout, stderr) with stdout and stderr as the byte buffers. To convert them to strings, use the 'decode(encoding="utf-8", errors="ignore")' function. The return code will be -1000 for a command timeout and -2000 for a stdout timeout. Args: command (str): The command to execute. command_timeout (float): The timeout of the whole command execution in seconds. output_timeout (float): The timeout of stdout/stderr outputs. Returns: The tuple (return_code, stdout_buffer, stderr_buffer) with the return code of the command (or -1000 on a command timeout resp. -2000 on a stdout timeout) and the stdout/stderr buffers as byte arrays. """ LOG.debug( "Execute command '%s' in shell environment with command timeout of %s seconds and " "output timeout of %s seconds.", command, command_timeout, output_timeout) stdout_r, stdout_w = pty.openpty() stderr_r, stderr_w = pty.openpty() proc = subprocess.Popen(command, bufsize=0, shell=True, stdout=stdout_w, stderr=stderr_w) os.close(stdout_w) os.close(stderr_w) output_start_time = time.time() command_start_time = output_start_time command_timeout_occured = False output_timeout_occured = False buffer_size = 1024 stdout_buffer = b'' stderr_buffer = b'' def read_buffer(read_fd, output_stream): buffer = b'' try: buffer = os.read(read_fd, buffer_size) except OSError as exception: if exception.errno != errno.EIO: raise else: output_stream.buffer.write(buffer) output_stream.buffer.flush() return buffer while proc.poll() is None: # Since the select call can't detect whether the process exited, we use a # small timeout on the select call (one second) and check afterwards for # a timeout on the stdout/stderr streams streams_ready = select.select([stdout_r, stderr_r], [], [], 1.0)[0] if streams_ready: if stdout_r in streams_ready: stdout_buffer += read_buffer(stdout_r, sys.stdout) if stderr_r in streams_ready: stderr_buffer += read_buffer(stderr_r, sys.stderr) output_start_time = time.time() elif proc.poll() is None: if output_timeout and ( (time.time() - output_start_time) >= output_timeout): LOG.debug( "No stdout/stderr output received within %d seconds!", (time.time() - output_start_time)) output_timeout_occured = True proc.kill() break if command_timeout and ( (time.time() - command_start_time) >= command_timeout): LOG.debug("Timeout occured after %d seconds!", (time.time() - command_start_time)) command_timeout_occured = True proc.kill() break # To cleanup any pending resources proc.wait() os.close(stdout_r) os.close(stderr_r) sys.stdout.buffer.flush() sys.stderr.buffer.flush() return_code = proc.returncode if command_timeout_occured: return_code = -1000 elif output_timeout_occured: return_code = -2000 LOG.debug("Command '%s' finished with return code %d.", command, return_code) return (return_code, stdout_buffer, stderr_buffer)
def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh', in_data=None): ''' run a command on the remote host ''' ssh_cmd = self._password_cmd() ssh_cmd += ["ssh", "-C"] if not in_data: ssh_cmd += ["-tt"] if utils.VERBOSITY > 3: ssh_cmd += ["-vvv"] else: ssh_cmd += ["-q"] ssh_cmd += self.common_args if self.ipv6: ssh_cmd += ['-6'] ssh_cmd += [self.host] if not self.runner.sudo or not sudoable: if executable: ssh_cmd.append(executable + ' -c ' + pipes.quote(cmd)) else: ssh_cmd.append(cmd) else: sudocmd, prompt, success_key = utils.make_sudo_cmd( sudo_user, executable, cmd) ssh_cmd.append(sudocmd) vvv("EXEC %s" % ssh_cmd, host=self.host) not_in_host_file = self.not_in_host_file(self.host) if C.HOST_KEY_CHECKING and not_in_host_file: # lock around the initial SSH connectivity so the user prompt about whether to add # the host to known hosts is not intermingled with multiprocess output. fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_EX) fcntl.lockf(self.runner.output_lockfile, fcntl.LOCK_EX) # create process if in_data: # do not use pseudo-pty p = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdin = p.stdin else: # try to use upseudo-pty try: # Make sure stdin is a proper (pseudo) pty to avoid: tcgetattr errors master, slave = pty.openpty() p = subprocess.Popen(ssh_cmd, stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdin = os.fdopen(master, 'w', 0) os.close(slave) except: p = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdin = p.stdin self._send_password() if self.runner.sudo and sudoable and self.runner.sudo_pass: fcntl.fcntl(p.stdout, fcntl.F_SETFL, fcntl.fcntl(p.stdout, fcntl.F_GETFL) | os.O_NONBLOCK) sudo_output = '' if in_data: # no terminal => no prompt on output. process is waiting for sudo_pass stdin.write(self.runner.sudo_pass + '\n') while not sudo_output.endswith( prompt) and success_key not in sudo_output: rfd, wfd, efd = select.select([p.stdout], [], [p.stdout], self.runner.timeout) if p.stdout in rfd: chunk = p.stdout.read() if not chunk: raise errors.AnsibleError( 'ssh connection closed waiting for sudo password prompt' ) sudo_output += chunk else: stdout = p.communicate() raise errors.AnsibleError( 'ssh connection error waiting for sudo password prompt' ) if success_key not in sudo_output: stdin.write(self.runner.sudo_pass + '\n') fcntl.fcntl(p.stdout, fcntl.F_SETFL, fcntl.fcntl(p.stdout, fcntl.F_GETFL) & ~os.O_NONBLOCK) # We can't use p.communicate here because the ControlMaster may have stdout open as well stdout = '' stderr = '' rpipes = [p.stdout, p.stderr] if in_data: try: stdin.write(in_data) stdin.close() except: raise errors.AnsibleError( 'SSH Error: data could not be sent to the remote host. Make sure this host can be reached over ssh' ) while True: rfd, wfd, efd = select.select(rpipes, [], rpipes, 1) # fail early if the sudo password is wrong if self.runner.sudo and sudoable and self.runner.sudo_pass: incorrect_password = gettext.dgettext("sudo", "Sorry, try again.") if stdout.endswith("%s\r\n%s" % (incorrect_password, prompt)): raise errors.AnsibleError('Incorrect sudo password') if p.stdout in rfd: dat = os.read(p.stdout.fileno(), 9000) stdout += dat if dat == '': rpipes.remove(p.stdout) if p.stderr in rfd: dat = os.read(p.stderr.fileno(), 9000) stderr += dat if dat == '': rpipes.remove(p.stderr) # only break out if we've emptied the pipes, or there is nothing to # read from and the process has finished. if (not rpipes or not rfd) and p.poll() is not None: break # Calling wait while there are still pipes to read can cause a lock elif not rpipes and p.poll() == None: p.wait() stdin.close( ) # close stdin after we read from stdout (see also issue #848) if C.HOST_KEY_CHECKING and not_in_host_file: # lock around the initial SSH connectivity so the user prompt about whether to add # the host to known hosts is not intermingled with multiprocess output. fcntl.lockf(self.runner.output_lockfile, fcntl.LOCK_UN) fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_UN) controlpersisterror = stderr.find( 'Bad configuration option: ControlPersist') != -1 or stderr.find( 'unknown configuration option: ControlPersist') != -1 if p.returncode != 0 and controlpersisterror: raise errors.AnsibleError( 'using -c ssh on certain older ssh versions may not support ControlPersist, set ANSIBLE_SSH_ARGS="" (or ansible_ssh_args in the config file) before running again' ) if p.returncode == 255 and in_data: raise errors.AnsibleError( 'SSH Error: data could not be sent to the remote host. Make sure this host can be reached over ssh' ) return (p.returncode, '', stdout, stderr)
def __init__(self, option_string=''): self.read_loop = True options = option_string.split(' ') cmd = ['iperf'] + options cmd_str = 'iperf ' + option_string master, slave = pty.openpty() self.process = Popen(cmd, stdout=slave, stderr=slave, close_fds=False) self.stdout = os.fdopen(master, 'r', 10000) # buffer which holds the iperf process output to read from self.readbuf = '' self.test_str = '' self.test_end = False # Prometheus export data # helper variables to calculate the metrics self.registry = CollectorRegistry() #buckets = (0.1, 0.2, 0.5, 1, 2, 5, 7, 10, 20, 50, 70, 90, float("inf")) self.prom_loss = Gauge('sonemu_packet_loss_percent', 'iperf packet loss (percent)', ['vnf_name'], registry=self.registry) self.prom_packets_loss = Gauge('sonemu_packets_loss_count', 'iperf packets lost (count)', ['vnf_name'], registry=self.registry) self.prom_packets_total = Gauge('sonemu_packets_total_count', 'iperf packets total (count)', ['vnf_name'], registry=self.registry) #buckets = (1, 9, 10, 11, 90, 100, 110, 900, 1000, 1100, float("inf")) self.prom_bandwith = Gauge('sonemu_bandwith_Mbitspersec', 'iperf bandwith (Mbits/sec)', ['vnf_name'], registry=self.registry) #buckets = (0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 5, 10, float("inf")) self.prom_jitter = Gauge('sonemu_jitter_ms', 'iperf jitter (ms)', ['vnf_name'], registry=self.registry) self.prom_bandwith.labels(vnf_name=vnf_name).set(float('nan')) self.prom_loss.labels(vnf_name=vnf_name).set(float('nan')) self.prom_packets_total.labels(vnf_name=vnf_name).set(float('nan')) self.prom_packets_loss.labels(vnf_name=vnf_name).set(float('nan')) self.prom_jitter.labels(vnf_name=vnf_name).set(float('nan')) while True: data = self.readline() if data: logging.info('stdout: {0}'.format(data)) self.parse_beginning_of_test(data) self.parse_end_of_test(data) if not self.test_end: bw = self.parse_bandwith(data) if not isnan(bw): self.prom_bandwith.labels(vnf_name=vnf_name).set(bw) else: self.prom_bandwith.labels(vnf_name=vnf_name).set(bw) # end of iperf test, no real measurement continue loss = self.parse_loss(data) self.prom_loss.labels(vnf_name=vnf_name).set(loss) lost, total = self.parse_packets(data) if lost and total: self.prom_packets_total.labels( vnf_name=vnf_name).set(total) self.prom_packets_loss.labels( vnf_name=vnf_name).set(lost) jitter = self.parse_jitter(data) self.prom_jitter.labels(vnf_name=vnf_name).set(jitter) else: self.prom_loss.labels(vnf_name=vnf_name).set(float('nan')) self.prom_jitter.labels(vnf_name=vnf_name).set(float('nan')) pushadd_to_gateway(pushgateway, job='sonemu-profile_sink', registry=self.registry)
def __init__(self, reactor, executable, args, environment, path, proto, uid=None, gid=None, usePTY=None): """ Spawn an operating-system process. This is where the hard work of disconnecting all currently open files / forking / executing the new process happens. (This is executed automatically when a Process is instantiated.) This will also run the subprocess as a given user ID and group ID, if specified. (Implementation Note: this doesn't support all the arcane nuances of setXXuid on UNIX: it will assume that either your effective or real UID is 0.) """ if pty is None and not isinstance(usePTY, (tuple, list)): # no pty module and we didn't get a pty to use raise NotImplementedError( "cannot use PTYProcess on platforms without the pty module.") abstract.FileDescriptor.__init__(self, reactor) _BaseProcess.__init__(self, proto) if isinstance(usePTY, (tuple, list)): masterfd, slavefd, ttyname = usePTY else: masterfd, slavefd = pty.openpty() ttyname = os.ttyname(slavefd) try: self._fork(path, uid, gid, executable, args, environment, masterfd=masterfd, slavefd=slavefd) except: if not isinstance(usePTY, (tuple, list)): os.close(masterfd) os.close(slavefd) raise # we are now in parent process: os.close(slavefd) fdesc.setNonBlocking(masterfd) self.fd = masterfd self.startReading() self.connected = 1 self.status = -1 try: self.proto.makeConnection(self) except: log.err() registerReapProcessHandler(self.pid, self)
def get_ports_pty(): master, slave = pty.openpty() s_name = os.ttyname(slave) return master, slave, s_name
def startShell(self, mnopts=None): """Start a shell process for running commands.""" if self.shell: error('shell is already running') return assert mnopts is None, 'mnopts not supported for DockerHost' self.container = '%s-%s' % (self.prefix, self.name) debug('Starting container %s with image "%s".' % (self.container, self.image)) self.kill(purge=True) container_tmp_dir = os.path.join(os.path.abspath(self.tmpdir), 'tmp') tmp_volume = container_tmp_dir + ':/tmp' base_cmd = ["docker", "run", "-ti", "--privileged", "--entrypoint", "env", "-h", self.name, "--name", self.container] opt_args = ['--net=%s' % self.network] env_vars = self.env_vars + ["TERM=dumb", "PS1=%s" % self.ps1] env_args = reduce(operator.add, (['--env', var] for var in env_vars), []) vol_args = reduce(operator.add, (['-v', var] for var in self.vol_maps), ['-v', tmp_volume]) image_args = [self.image, "bash", "--norc", "-is", "mininet:" + self.name] cmd = base_cmd + opt_args + env_args + vol_args + image_args self.master, self.slave = pty.openpty() debug('docker command "%s", fd %d, fd %d' % (' '.join(cmd), self.master, self.slave)) try: self.shell = self._popen(cmd, stdin=self.slave, stdout=self.slave, stderr=self.slave) self.stdin = os.fdopen(self.master, 'r') self.stdout = self.stdin self.pollOut = select.poll() # pylint: disable=invalid-name self.pollOut.register(self.stdout) # pylint: disable=no-member self.outToNode[self.stdout.fileno()] = self # pylint: disable=no-member self.pollIn = select.poll() # pylint: disable=invalid-name self.pollIn.register(self.stdout, select.POLLIN) # pylint: disable=no-member self.inToNode[self.stdin.fileno()] = self # pylint: disable=no-member self.execed = False self.lastCmd = None # pylint: disable=invalid-name self.lastPid = None # pylint: disable=invalid-name self.readbuf = '' self.waiting = True data = '' while True: data = self.read(1) if data[-1] == self.ps1: break self.readbuf = '' self.waiting = False except Exception: error('docker cmd: %s' % ' '.join(cmd)) if self.shell.returncode: error('returncode: %d' % self.shell.returncode) if self.shell: self.shell.poll() raise self.pid = self.inspect_pid() debug("Container %s created pid %s/%s." % (self.container, self.pid, self.shell.pid)) self.cmd('unset HISTFILE; stty -echo; set +m') # pylint: disable=no-member
def run( cmd, address=None, service_type=None, name=None, user=None, user_mode=os.getuid() != 0, nice=None, runtime_max_sec=None, env=None, extra=None, cwd=None, machine=None, wait=False, remain_after_exit=False, collect=False, raise_on_fail=False, pty=None, pty_master=None, pty_path=None, stdin=None, stdout=None, stderr=None, _wait_polling=None, slice_=None, stop_cmd=None, stop_post_cmd=None, start_pre_cmd=None, start_post_cmd=None, ): """ pystemd.run imitates systemd-run, but with a pythonic feel to it. Options: cmd: Array with the command to execute (absolute path only) stop_cmd: Array with the command to execute on stop (absolute path only) stop_post_cmd: Array with the command to execute after stop (absolute path only) start_pre_cmd: Array with the command to execute on pre start (absolute path only) start_post_cmd: Array with the command to execute on on post start (absolute path only) address: A custom dbus socket address service_type: Set the unit type, e.g. notify, oneshot. If you dont give a value, the unit type will be whatever systemd thinks is the default. name: Name of the unit. If not provided, it will be autogenerated. user: Username to execute the command, defaults to current user. user_mode: Equivalent to running `systemd-run --user`. Defaults to True if current user id not root (uid = 0). nice: Nice level to run the command. runtime_max_sec: Set seconds before sending a sigterm to the process, if the service does not die nicely, it will send a sigkill. env: A dict with environment variables. extra: If you know what you are doing, you can pass extra configuration settings to the start_transient_unit method. machine: Machine name to execute the command, by default we connect to the host's dbus. wait: Wait for command completion before returning control, defaults to False. remain_after_exit: If True, the transient unit will remain after cmd has finished, also if true, this methods will return pystemd.systemd1.Unit object. defaults to False and this method returns None and the unit will be gone as soon as is done. collect: Unload unit after it ran, even when failed. raise_on_fail: Will raise a PystemdRunError is cmd exit with non 0 status code, it won't take affect unless you set wait=True, defaults to False. pty: Set this variable to True if you want a pty to be created. if you pass a `machine`, the pty will be created in the machine. Setting this value will ignore whatever you set in pty_master and pty_path. pty_master: It has only meaning if you pass a pty_path also, this file descriptor will be used to forward redirection to `stdin` and `stdout` if no `stdin` or `stdout` is present, then this value does nothing. pty_path: Setting this value will pass this pty_path to the created process and will connect the process stdin, stdout and stderr to this pty. by itself it only ensure that your process has a real pty that can have ioctl operation over it. if you also pass a `pty_master`, `stdin` and `stdout` the pty forwars is handle for you. stdin: Specify a file descriptor for stdin. By default this is `None` and your unit will not have a stdin. If you set pty = True, or set a `pty_master` then that pty will be read and forwarded to this file descriptor. stdout: Specify a file descriptor for stdout. By default this is `None` and your unit will not have a stdout. If you set pty = True, or set a `pty_master` then that pty will be read and forwarded to this file descriptor. stderr: Specify a file descriptor for stderr. By default this is `None` and your unit will not have a stderr. slice_: the slice under you want to run the unit. More info and examples in: https://github.com/facebookincubator/pystemd/blob/master/_docs/pystemd.run.md """ def bus_factory(): if address: return DBusAddress(x2char_star(address)) elif machine: return DBusMachine(x2char_star(machine)) else: return DBus(user_mode=user_mode) name = x2char_star(name or "pystemd{}.service".format(uuid.uuid4().hex)) runtime_max_usec = (runtime_max_sec or 0) * 10**6 or runtime_max_sec stdin, stdout, stderr = get_fno(stdin), get_fno(stdout), get_fno(stderr) env = env or {} unit_properties = {} selectors = [] extra = extra or {} start_cmd = x2cmdlist(cmd, False) + extra.pop(b"ExecStart", []) stop_cmd = x2cmdlist(stop_cmd, False) + extra.pop(b"ExecStop", []) stop_post_cmd = x2cmdlist(stop_post_cmd, False) + extra.pop(b"ExecStopPost", []) start_pre_cmd = x2cmdlist(start_pre_cmd, False) + extra.pop(b"ExecStartPre", []) start_post_cmd = x2cmdlist(start_post_cmd, False) + extra.pop(b"ExecStartPost", []) if user_mode: _wait_polling = _wait_polling or 0.5 with CExit() as ctexit, bus_factory() as bus, SDManager(bus=bus) as manager: if pty: if machine: with pystemd.machine1.Machine(machine) as m: pty_master, pty_path = m.Machine.OpenPTY() else: pty_master, pty_follower = ptylib.openpty() pty_path = os.ttyname(pty_follower).encode() ctexit.register(os.close, pty_master) if slice_: unit_properties[b"Slice"] = x2char_star(slice_) if pty_path: unit_properties.update( { b"StandardInput": b"tty", b"StandardOutput": b"tty", b"StandardError": b"tty", b"TTYPath": pty_path, } ) if None not in (stdin, pty_master): # lets set raw mode for stdin so we can forward input without # waiting for a new line, but lets also make sure we return the # attributes as they where after this method is done stdin_attrs = tty.tcgetattr(stdin) tty.setraw(stdin) ctexit.register(tty.tcsetattr, stdin, tty.TCSAFLUSH, stdin_attrs) selectors.append(stdin) if None not in (stdout, pty_master): if os.getenv("TERM"): env[b"TERM"] = env.get(b"TERM", os.getenv("TERM").encode()) selectors.append(pty_master) # lets be a friend and set the size of the pty. winsize = fcntl.ioctl( stdout, termios.TIOCGWINSZ, struct.pack("HHHH", 0, 0, 0, 0) ) fcntl.ioctl(pty_master, termios.TIOCSWINSZ, winsize) else: unit_properties.update( { b"StandardInputFileDescriptor": get_fno(stdin) if stdin else stdin, b"StandardOutputFileDescriptor": get_fno(stdout) if stdout else stdout, b"StandardErrorFileDescriptor": get_fno(stderr) if stderr else stderr, } ) unit_properties.update( { b"Type": service_type, b"Description": b"pystemd: " + name, b"ExecStartPre": start_pre_cmd or None, b"ExecStart": start_cmd, b"ExecStartPost": start_post_cmd or None, b"ExecStop": stop_cmd or None, b"ExecStopPost": stop_post_cmd or None, b"RemainAfterExit": remain_after_exit, b"CollectMode": b"inactive-or-failed" if collect else None, b"WorkingDirectory": cwd, b"User": user, b"Nice": nice, b"RuntimeMaxUSec": runtime_max_usec, b"Environment": [ b"%s=%s" % (x2char_star(key), x2char_star(value)) for key, value in env.items() ] or None, } ) unit_properties.update(extra) unit_properties = {k: v for k, v in unit_properties.items() if v is not None} unit = Unit(name, bus=bus, _autoload=True) if wait: mstr = ( ( "type='signal'," "sender='org.freedesktop.systemd1'," "path='{}'," "interface='org.freedesktop.DBus.Properties'," "member='PropertiesChanged'" ) .format(unit.path.decode()) .encode() ) monbus = bus_factory() monbus.open() ctexit.register(monbus.close) monitor = pystemd.DBus.Manager(bus=monbus, _autoload=True) monitor.Monitoring.BecomeMonitor([mstr], 0) monitor_fd = monbus.get_fd() selectors.append(monitor_fd) # start the process unit_start_job = manager.Manager.StartTransientUnit( name, b"fail", unit_properties ) while wait: _in, _, _ = select.select(selectors, [], [], _wait_polling) if stdin in _in: data = os.read(stdin, 1024) os.write(pty_master, data) if pty_master in _in: try: data = os.read(pty_master, 1024) except OSError: selectors.remove(pty_master) else: os.write(stdout, data) if monitor_fd in _in: m = monbus.process() if m.is_empty(): continue m.process_reply(False) if ( m.get_path() == unit.path and m.body[0] == b"org.freedesktop.systemd1.Unit" ): _, message_job_path = m.body[1].get(b"Job", (0, b"/")) if ( message_job_path != unit_start_job and m.body[1].get(b"SubState") in EXIT_SUBSTATES ): break if _wait_polling and not _in and unit.Service.MainPID == 0: # on usermode the subscribe to events does not work that well # this is a temporary hack. you can always not wait on usermode. break if raise_on_fail: if unit.Service.ExecMainStatus: raise PystemdRunError( "cmd {} exited with status {}".format( cmd, unit.Service.ExecMainStatus ) ) unit.load() unit.bus_context = bus_factory return unit
def main(**args): # figure out explicit defines defines = {} for define in args['D']: k, v, *_ = define.split('=', 2) + [''] defines[k] = v # and what class of TestCase to run classes = [] if args.get('normal'): classes.append(TestCase) if args.get('reentrant'): classes.append(ReentrantTestCase) if args.get('valgrind'): classes.append(ValgrindTestCase) if not classes: classes = [TestCase] suites = [] for testpath in args['test_paths']: # optionally specified test case/perm testpath, *filter = testpath.split('#') filter = [int(f) for f in filter] # figure out the suite's toml file if os.path.isdir(testpath): testpath = testpath + '/*.toml' elif os.path.isfile(testpath): testpath = testpath elif testpath.endswith('.toml'): testpath = TEST_PATHS + '/' + testpath else: testpath = TEST_PATHS + '/' + testpath + '.toml' # find tests for path in glob.glob(testpath): suites.append(TestSuite(path, classes, defines, filter, **args)) # sort for reproducability suites = sorted(suites) # generate permutations for suite in suites: suite.permute(**args) # build tests in parallel print('====== building ======') makefiles = [] targets = [] for suite in suites: makefile, target = suite.build(**args) makefiles.append(makefile) targets.append(target) cmd = (['make', '-f', 'Makefile'] + list(it.chain.from_iterable(['-f', m] for m in makefiles)) + [target for target in targets]) mpty, spty = pty.openpty() if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, stdout=spty, stderr=spty) os.close(spty) mpty = os.fdopen(mpty, 'r', 1) stdout = [] while True: try: line = mpty.readline() except OSError as e: if e.errno == errno.EIO: break raise stdout.append(line) if args.get('verbose'): sys.stdout.write(line) # intercept warnings m = re.match( '^{0}([^:]+):(\d+):(?:\d+:)?{0}{1}:{0}(.*)$'.format( '(?:\033\[[\d;]*.| )*', 'warning'), line) if m and not args.get('verbose'): try: with open(m.group(1)) as f: lineno = int(m.group(2)) line = next(it.islice(f, lineno - 1, None)).strip('\n') sys.stdout.write( "\033[01m{path}:{lineno}:\033[01;35mwarning:\033[m " "{message}\n{line}\n\n".format(path=m.group(1), line=line, lineno=lineno, message=m.group(3))) except: pass proc.wait() if proc.returncode != 0: if not args.get('verbose'): for line in stdout: sys.stdout.write(line) sys.exit(-1) print('built %d test suites, %d test cases, %d permutations' % (len(suites), sum(len(suite.cases) for suite in suites), sum(len(suite.perms) for suite in suites))) total = 0 for suite in suites: for perm in suite.perms: total += perm.shouldtest(**args) if total != sum(len(suite.perms) for suite in suites): print('filtered down to %d permutations' % total) # only requested to build? if args.get('build'): return 0 print('====== testing ======') try: for suite in suites: suite.test(**args) except TestFailure: pass print('====== results ======') passed = 0 failed = 0 for suite in suites: for perm in suite.perms: if perm.result == PASS: passed += 1 elif isinstance(perm.result, TestFailure): sys.stdout.write( "\033[01m{path}:{lineno}:\033[01;31mfailure:\033[m " "{perm} failed\n".format(perm=perm, path=perm.suite.path, lineno=perm.lineno, returncode=perm.result.returncode or 0)) if perm.result.stdout: if perm.result.assert_: stdout = perm.result.stdout[:-1] else: stdout = perm.result.stdout for line in stdout[-5:]: sys.stdout.write(line) if perm.result.assert_: sys.stdout.write( "\033[01m{path}:{lineno}:\033[01;31massert:\033[m " "{message}\n{line}\n".format(**perm.result.assert_)) sys.stdout.write('\n') failed += 1 if args.get('coverage'): # collect coverage info # why -j1? lcov doesn't work in parallel because of gcov limitations cmd = (['make', '-j1', '-f', 'Makefile'] + list(it.chain.from_iterable(['-f', m] for m in makefiles)) + (['COVERAGETARGET=%s' % args['coverage']] if isinstance( args['coverage'], str) else []) + [ suite.path + '.info' for suite in suites if any(perm.result == PASS for perm in suite.perms) ]) if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, stdout=sp.PIPE if not args.get('verbose') else None, stderr=sp.STDOUT if not args.get('verbose') else None, universal_newlines=True) proc.wait() if proc.returncode != 0: if not args.get('verbose'): for line in proc.stdout: sys.stdout.write(line) sys.exit(-1) if args.get('gdb'): failure = None for suite in suites: for perm in suite.perms: if isinstance(perm.result, TestFailure): failure = perm.result if failure is not None: print('======= gdb ======') # drop into gdb failure.case.test(failure=failure, **args) sys.exit(0) print('tests passed %d/%d (%.2f%%)' % (passed, total, 100 * (passed / total if total else 1.0))) print('tests failed %d/%d (%.2f%%)' % (failed, total, 100 * (failed / total if total else 1.0))) return 1 if failed > 0 else 0
def mk(name): a, b = openpty() masters.append(a) slaves.append(b) names.append(name)
def _run(self, cmd, in_data, sudoable=True, checkrc=True): ''' Starts the command and communicates with it until it ends. ''' display_cmd = list(map(shlex_quote, map(to_text, cmd))) display.vvv(u'SSH: EXEC {0}'.format(u' '.join(display_cmd)), host=self.host) # Start the given command. If we don't need to pipeline data, we can try # to use a pseudo-tty (ssh will have been invoked with -tt). If we are # pipelining data, or can't create a pty, we fall back to using plain # old pipes. p = None if isinstance(cmd, (text_type, binary_type)): cmd = to_bytes(cmd) else: cmd = list(map(to_bytes, cmd)) if not in_data: try: # Make sure stdin is a proper pty to avoid tcgetattr errors master, slave = pty.openpty() if PY3 and self._play_context.password: p = subprocess.Popen(cmd, stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE, pass_fds=self.sshpass_pipe) else: p = subprocess.Popen(cmd, stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdin = os.fdopen(master, 'wb', 0) os.close(slave) except (OSError, IOError): p = None if not p: if PY3 and self._play_context.password: p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, pass_fds=self.sshpass_pipe) else: p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdin = p.stdin # If we are using SSH password authentication, write the password into # the pipe we opened in _build_command. if self._play_context.password: os.close(self.sshpass_pipe[0]) try: os.write(self.sshpass_pipe[1], to_bytes(self._play_context.password) + b'\n') except OSError as e: # Ignore broken pipe errors if the sshpass process has exited. if e.errno != errno.EPIPE or p.poll() is None: raise os.close(self.sshpass_pipe[1]) # # SSH state machine # # Now we read and accumulate output from the running process until it # exits. Depending on the circumstances, we may also need to write an # escalation password and/or pipelined input to the process. states = [ 'awaiting_prompt', 'awaiting_escalation', 'ready_to_send', 'awaiting_exit' ] # Are we requesting privilege escalation? Right now, we may be invoked # to execute sftp/scp with sudoable=True, but we can request escalation # only when using ssh. Otherwise we can send initial data straightaway. state = states.index('ready_to_send') if b'ssh' in cmd: if self._play_context.prompt: # We're requesting escalation with a password, so we have to # wait for a password prompt. state = states.index('awaiting_prompt') display.debug(u'Initial state: %s: %s' % (states[state], self._play_context.prompt)) elif self._play_context.become and self._play_context.success_key: # We're requesting escalation without a password, so we have to # detect success/failure before sending any initial data. state = states.index('awaiting_escalation') display.debug(u'Initial state: %s: %s' % (states[state], self._play_context.success_key)) # We store accumulated stdout and stderr output from the process here, # but strip any privilege escalation prompt/confirmation lines first. # Output is accumulated into tmp_*, complete lines are extracted into # an array, then checked and removed or copied to stdout or stderr. We # set any flags based on examining the output in self._flags. b_stdout = b_stderr = b'' b_tmp_stdout = b_tmp_stderr = b'' self._flags = dict( become_prompt=False, become_success=False, become_error=False, become_nopasswd_error=False ) # select timeout should be longer than the connect timeout, otherwise # they will race each other when we can't connect, and the connect # timeout usually fails timeout = 2 + self._play_context.timeout rpipes = [p.stdout, p.stderr] for fd in rpipes: fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK) # If we can send initial data without waiting for anything, we do so # before we call select. if states[state] == 'ready_to_send' and in_data: self._send_initial_data(stdin, in_data) state += 1 while True: rfd, wfd, efd = select.select(rpipes, [], [], timeout) # We pay attention to timeouts only while negotiating a prompt. if not rfd: if state <= states.index('awaiting_escalation'): # If the process has already exited, then it's not really a # timeout; we'll let the normal error handling deal with it. if p.poll() is not None: break self._terminate_process(p) raise AnsibleError('Timeout (%ds) waiting for privilege escalation prompt: %s' % (timeout, to_native(b_stdout))) # Read whatever output is available on stdout and stderr, and stop # listening to the pipe if it's been closed. if p.stdout in rfd: b_chunk = p.stdout.read() if b_chunk == b'': rpipes.remove(p.stdout) # When ssh has ControlMaster (+ControlPath/Persist) enabled, the # first connection goes into the background and we never see EOF # on stderr. If we see EOF on stdout, lower the select timeout # to reduce the time wasted selecting on stderr if we observe # that the process has not yet existed after this EOF. Otherwise # we may spend a long timeout period waiting for an EOF that is # not going to arrive until the persisted connection closes. timeout = 1 b_tmp_stdout += b_chunk display.debug("stdout chunk (state=%s):\n>>>%s<<<\n" % (state, to_text(b_chunk))) if p.stderr in rfd: b_chunk = p.stderr.read() if b_chunk == b'': rpipes.remove(p.stderr) b_tmp_stderr += b_chunk display.debug("stderr chunk (state=%s):\n>>>%s<<<\n" % (state, to_text(b_chunk))) # We examine the output line-by-line until we have negotiated any # privilege escalation prompt and subsequent success/error message. # Afterwards, we can accumulate output without looking at it. if state < states.index('ready_to_send'): if b_tmp_stdout: b_output, b_unprocessed = self._examine_output('stdout', states[state], b_tmp_stdout, sudoable) b_stdout += b_output b_tmp_stdout = b_unprocessed if b_tmp_stderr: b_output, b_unprocessed = self._examine_output('stderr', states[state], b_tmp_stderr, sudoable) b_stderr += b_output b_tmp_stderr = b_unprocessed else: b_stdout += b_tmp_stdout b_stderr += b_tmp_stderr b_tmp_stdout = b_tmp_stderr = b'' # If we see a privilege escalation prompt, we send the password. # (If we're expecting a prompt but the escalation succeeds, we # didn't need the password and can carry on regardless.) if states[state] == 'awaiting_prompt': if self._flags['become_prompt']: display.debug('Sending become_pass in response to prompt') stdin.write(to_bytes(self._play_context.become_pass) + b'\n') self._flags['become_prompt'] = False state += 1 elif self._flags['become_success']: state += 1 # We've requested escalation (with or without a password), now we # wait for an error message or a successful escalation. if states[state] == 'awaiting_escalation': if self._flags['become_success']: display.debug('Escalation succeeded') self._flags['become_success'] = False state += 1 elif self._flags['become_error']: display.debug('Escalation failed') self._terminate_process(p) self._flags['become_error'] = False raise AnsibleError('Incorrect %s password' % self._play_context.become_method) elif self._flags['become_nopasswd_error']: display.debug('Escalation requires password') self._terminate_process(p) self._flags['become_nopasswd_error'] = False raise AnsibleError('Missing %s password' % self._play_context.become_method) elif self._flags['become_prompt']: # This shouldn't happen, because we should see the "Sorry, # try again" message first. display.debug('Escalation prompt repeated') self._terminate_process(p) self._flags['become_prompt'] = False raise AnsibleError('Incorrect %s password' % self._play_context.become_method) # Once we're sure that the privilege escalation prompt, if any, has # been dealt with, we can send any initial data and start waiting # for output. if states[state] == 'ready_to_send': if in_data: self._send_initial_data(stdin, in_data) state += 1 # Now we're awaiting_exit: has the child process exited? If it has, # and we've read all available output from it, we're done. if p.poll() is not None: if not rpipes or not rfd: break # We should not see further writes to the stdout/stderr file # descriptors after the process has closed, set the select # timeout to gather any last writes we may have missed. timeout = 0 continue # If the process has not yet exited, but we've already read EOF from # its stdout and stderr (and thus removed both from rpipes), we can # just wait for it to exit. elif not rpipes: p.wait() break # Otherwise there may still be outstanding data to read. # close stdin after process is terminated and stdout/stderr are read # completely (see also issue #848) stdin.close() if C.HOST_KEY_CHECKING: if cmd[0] == b"sshpass" and p.returncode == 6: raise AnsibleError('Using a SSH password instead of a key is not possible because Host Key checking is enabled and sshpass does not support this. Please add this host\'s fingerprint to your known_hosts file to manage this host.') controlpersisterror = b'Bad configuration option: ControlPersist' in b_stderr or b'unknown configuration option: ControlPersist' in b_stderr if p.returncode != 0 and controlpersisterror: raise AnsibleError('using -c ssh on certain older ssh versions may not support ControlPersist, set ANSIBLE_SSH_ARGS="" (or ssh_args in [ssh_connection] section of the config file) before running again') if p.returncode == 255 and in_data and checkrc: raise AnsibleConnectionFailure('SSH Error: data could not be sent to the remote host. Make sure this host can be reached over ssh') return (p.returncode, b_stdout, b_stderr)
def run(cmd, print_error=True, asynchronous=False, stdin=False, stderr=subprocess.STDOUT, outfile=None, env_vars=None, inherit_cwd=False, inherit_env=True, tty=False): env_dict = os.environ.copy() if inherit_env else {} if env_vars: env_dict.update(env_vars) env_dict = dict([(k, to_str(str(v))) for k, v in env_dict.items()]) if tty: asynchronous = True stdin = True try: cwd = os.getcwd() if inherit_cwd else None if not asynchronous: if stdin: return subprocess.check_output(cmd, shell=True, stderr=stderr, env=env_dict, stdin=subprocess.PIPE, cwd=cwd) output = subprocess.check_output(cmd, shell=True, stderr=stderr, env=env_dict, cwd=cwd) return output.decode(config.DEFAULT_ENCODING) stdin_arg = subprocess.PIPE if stdin else None stdout_arg = open(outfile, 'ab') if isinstance( outfile, six.string_types) else outfile stderr_arg = stderr if tty: # Note: leave the "pty" import here (not supported in Windows) import pty master_fd, slave_fd = pty.openpty() stdin_arg = slave_fd stdout_arg = stderr_arg = None # start the actual sub process kwargs = {} if is_linux() or is_mac_os(): kwargs['preexec_fn'] = os.setsid process = subprocess.Popen(cmd, shell=True, stdin=stdin_arg, bufsize=-1, stderr=stderr_arg, stdout=stdout_arg, env=env_dict, cwd=cwd, **kwargs) if tty: # based on: https://stackoverflow.com/questions/41542960 def pipe_streams(*args): while process.poll() is None: r, w, e = select.select([sys.stdin, master_fd], [], []) if sys.stdin in r: d = os.read(sys.stdin.fileno(), 10240) os.write(master_fd, d) elif master_fd in r: o = os.read(master_fd, 10240) if o: os.write(sys.stdout.fileno(), o) FuncThread(pipe_streams).start() return process except subprocess.CalledProcessError as e: if print_error: print("ERROR: '%s': exit code %s; output: %s" % (cmd, e.returncode, e.output)) sys.stdout.flush() raise e
def check_output(self, command, *, print_on_silent_log=False): """Run a command and supply the output to callback functions""" logger = logging.getLogger("Process") res = [] mfd, sfd = pty.openpty() flags = fcntl.fcntl(mfd, fcntl.F_GETFL) fcntl.fcntl(mfd, fcntl.F_SETFL, flags | os.O_NONBLOCK) process = subprocess.Popen(command, stderr=sfd, stdout=sfd, bufsize=0) logger.log(ProcessWrapper.loglevel, "[%d] command: %s", process.pid, " ".join(command)) # do not register/unregister already registered print_callback if ProcessWrapper.print_callback in self.callbacks: print_on_silent_log = False if print_on_silent_log and logger.getEffectiveLevel( ) > ProcessWrapper.loglevel: self.enable_print() # close sfd so we notice when the child is gone os.close(sfd) # get a file object from the fd buf = b"" while True: try: raw = os.read(mfd, 4096) except BlockingIOError as ex: # wait for new data and retry select.select([mfd], [], [mfd], 0.1) continue except OSError as e: if e.errno == 5: break raise if raw: buf += raw *parts, buf = buf.split(b'\r') res.extend(parts) for part in parts: for callback in self.callbacks: callback(part, process) process.poll() if process.returncode is not None: break os.close(mfd) process.wait() if buf: # process incomplete line res.append(buf) if buf[-1] != b'\n': buf += b'\n' for callback in self.callbacks: callback(buf, process) if print_on_silent_log and logger.getEffectiveLevel( ) > ProcessWrapper.loglevel: self.disable_print() if process.returncode != 0: raise subprocess.CalledProcessError(process.returncode, command, output=b'\r'.join(res)) # this converts '\r\n' to '\n' to be more compatible to the behaviour # of the normal subprocess module return b'\n'.join([r.strip(b'\n') for r in res])
def test(self, exec=[], persist=False, cycles=None, gdb=False, failure=None, disk=None, **args): # build command cmd = exec + [ './%s.test' % self.suite.path, repr(self.caseno), repr(self.permno) ] # persist disk or keep in RAM for speed? if persist: if not disk: disk = self.suite.path + '.disk' if persist != 'noerase': try: with open(disk, 'w') as f: f.truncate(0) if args.get('verbose'): print('truncate --size=0', disk) except FileNotFoundError: pass cmd.append(disk) # simulate power-loss after n cycles? if cycles: cmd.append(str(cycles)) # failed? drop into debugger? if gdb and failure: ncmd = ['gdb'] if gdb == 'assert': ncmd.extend(['-ex', 'r']) if failure.assert_: ncmd.extend(['-ex', 'up 2']) elif gdb == 'main': ncmd.extend([ '-ex', 'b %s:%d' % (self.suite.path, self.code_lineno), '-ex', 'r' ]) ncmd.extend(['--args'] + cmd) if args.get('verbose'): print(' '.join(shlex.quote(c) for c in ncmd)) signal.signal(signal.SIGINT, signal.SIG_IGN) sys.exit(sp.call(ncmd)) # run test case! mpty, spty = pty.openpty() if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, stdout=spty, stderr=spty) os.close(spty) mpty = os.fdopen(mpty, 'r', 1) stdout = [] assert_ = None try: while True: try: line = mpty.readline() except OSError as e: if e.errno == errno.EIO: break raise stdout.append(line) if args.get('verbose'): sys.stdout.write(line) # intercept asserts m = re.match( '^{0}([^:]+):(\d+):(?:\d+:)?{0}{1}:{0}(.*)$'.format( '(?:\033\[[\d;]*.| )*', 'assert'), line) if m and assert_ is None: try: with open(m.group(1)) as f: lineno = int(m.group(2)) line = (next(it.islice(f, lineno - 1, None)).strip('\n')) assert_ = { 'path': m.group(1), 'line': line, 'lineno': lineno, 'message': m.group(3) } except: pass except KeyboardInterrupt: raise TestFailure(self, 1, stdout, None) proc.wait() # did we pass? if proc.returncode != 0: raise TestFailure(self, proc.returncode, stdout, assert_) else: return PASS