def run_proc(proc, retcode, timeout=None):
"""Waits for the given process to terminate, with the expected exit code
:param proc: a running Popen-like object
:param retcode: the expected return (exit) code of the process. It defaults to 0 (the
convention for success). If ``None``, the return code is ignored.
It may also be a tuple (or any object that supports ``__contains__``)
of expected return codes.
:param timeout: the number of seconds (a ``float``) to allow the process to run, before
forcefully terminating it. If ``None``, not timeout is imposed; otherwise
the process is expected to terminate within that timeout value, or it will
be killed and :class:`ProcessTimedOut <plumbum.cli.ProcessTimedOut>`
will be raised
:returns: A tuple of (return code, stdout, stderr)
"""
_register_proc_timeout(proc, timeout)
stdout, stderr = proc.communicate()
proc._end_time = time.time()
if not stdout:
stdout = six.b("")
if not stderr:
stderr = six.b("")
if getattr(proc, "encoding", None):
stdout = stdout.decode(proc.encoding, "ignore")
stderr = stderr.decode(proc.encoding, "ignore")
return _check_process(proc, retcode, timeout, stdout, stderr)
def _iter_lines(proc, decode, linesize):
try:
from selectors import DefaultSelector, EVENT_READ
except ImportError:
# Pre Python 3.4 implementation
from select import select
def selector():
while True:
rlist, _, _ = select([proc.stdout.channel], [], [])
for _ in rlist:
yield
else:
# Python 3.4 implementation
def selector():
sel = DefaultSelector()
sel.register(proc.stdout.channel, EVENT_READ)
while True:
for key, mask in sel.select():
yield
for _ in selector():
if proc.stdout.channel.recv_ready():
yield 0, decode(six.b(proc.stdout.readline(linesize)))
if proc.stdout.channel.recv_stderr_ready():
yield 1, decode(six.b(proc.stderr.readline(linesize)))
if proc.poll() is not None:
break
for line in proc.stdout:
yield 0, decode(six.b(line))
for line in proc.stderr:
yield 1, decode(six.b(line))
def test_atomic_file(self):
af1 = AtomicFile("tmp.txt")
af2 = AtomicFile("tmp.txt")
af1.write_atomic(six.b("foo"))
af2.write_atomic(six.b("bar"))
self.assertEqual(af1.read_atomic(), six.b("bar"))
self.assertEqual(af2.read_atomic(), six.b("bar"))
local.path("tmp.txt").delete()
def get_pe_subsystem(filename):
with open(filename, "rb") as f:
if f.read(2) != six.b("MZ"):
return None
f.seek(LFANEW_OFFSET)
lfanew = struct.unpack("L", f.read(4))[0]
f.seek(lfanew)
if f.read(4) != six.b("PE\x00\x00"):
return None
f.seek(FILE_HEADER_SIZE + SUBSYSTEM_OFFSET, 1)
subsystem = struct.unpack("H", f.read(2))[0]
return subsystem
def readline(self):
"""Reads the next line from the pipe; returns "" when the special marker is reached.
Raises ``EOFError`` if the underlying pipe has closed"""
if self.pipe is None:
return six.b("")
line = self.pipe.readline()
if not line:
raise EOFError()
if line.strip() == self.marker:
self.pipe = None
line = six.b("")
return line
def communicate(self, input=None):
"""Consumes the process' stdout and stderr until the it terminates.
:param input: An optional bytes/buffer object to send to the process over stdin
:returns: A tuple of (stdout, stderr)
"""
stdout = []
stderr = []
sources = [("1", stdout, self.stdout)]
if not self.isatty:
# in tty mode, stdout and stderr are unified
sources.append(("2", stderr, self.stderr))
i = 0
while sources:
if input:
chunk = input[:1000]
self.stdin.write(chunk)
self.stdin.flush()
input = input[1000:]
i = (i + 1) % len(sources)
name, coll, pipe = sources[i]
try:
line = pipe.readline()
shell_logger.debug("%s> %r", name, line)
except EOFError:
shell_logger.debug("%s> Nothing returned.", name)
self.proc.poll()
returncode = self.proc.returncode
if returncode == 5:
raise IncorrectLogin(
"Incorrect username or password provided")
elif returncode == 6:
raise HostPublicKeyUnknown(
"The authenticity of the host can't be established")
msg = "No communication channel detected. Does the remote exist?"
msgerr = "No stderr result detected. Does the remote have Bash as the default shell?"
raise SSHCommsChannel2Error(
msgerr) if name == "2" else SSHCommsError(msg)
if not line:
del sources[i]
else:
coll.append(line)
if self.isatty:
stdout.pop(0) # discard first line of prompt
try:
self.returncode = int(stdout.pop(-1))
except (IndexError, ValueError):
self.returncode = "Unknown"
self._done = True
stdout = six.b("").join(stdout)
stderr = six.b("").join(stderr)
return stdout, stderr
def test_tunnel(self):
with self._connect() as rem:
p = rem.python["-c", self.TUNNEL_PROG].popen()
try:
port = int(p.stdout.readline().strip())
except ValueError:
print(p.communicate())
raise
s = rem.connect_sock(port)
s.send(six.b("world"))
data = s.recv(100)
s.close()
self.assertEqual(data, six.b("hello world"))
def test_read_write(self):
with local.tempdir() as dir:
f = dir / "test.txt"
text = six.b('hello world\xd7\xa9\xd7\x9c\xd7\x95\xd7\x9d').decode("utf8")
f.write(text, "utf8")
text2 = f.read("utf8")
self.assertEqual(text, text2)
def popen(self, cmd):
"""Runs the given command in the shell, adding some decoration around it. Only a single
command can be executed at any given time.
:param cmd: The command (string or :class:`Command <plumbum.commands.BaseCommand>` object)
to run
:returns: A :class:`SessionPopen <plumbum.session.SessionPopen>` instance
"""
if self.proc is None:
raise ShellSessionError("Shell session has already been closed")
if self._current and not self._current._done:
raise ShellSessionError("Each shell may start only one process at a time")
if isinstance(cmd, BaseCommand):
full_cmd = cmd.formulate(1)
else:
full_cmd = cmd
marker = "--.END%s.--" % (time.time() * random.random(),)
if full_cmd.strip():
full_cmd += " ; "
else:
full_cmd = "true ; "
full_cmd += "echo $? ; echo '%s'" % (marker,)
if not self.isatty:
full_cmd += " ; echo '%s' 1>&2" % (marker,)
if self.encoding:
full_cmd = full_cmd.encode(self.encoding)
shell_logger.debug("Running %r", full_cmd)
self.proc.stdin.write(full_cmd + six.b("\n"))
self.proc.stdin.flush()
self._current = SessionPopen(full_cmd, self.isatty, self.proc.stdin,
MarkedPipe(self.proc.stdout, marker), MarkedPipe(self.proc.stderr, marker),
self.encoding, self.proc.machine)
return self._current
def test_chown(self):
with local.tempdir() as dir:
p = dir / "foo.txt"
p.write(six.b("hello"))
self.assertEqual(p.uid, os.getuid())
self.assertEqual(p.gid, os.getgid())
p.chown(p.uid.name)
self.assertEqual(p.uid, os.getuid())
def _read_all(self):
self._fileobj.seek(0)
data = []
while True:
buf = self._fileobj.read(self.CHUNK_SIZE)
data.append(buf)
if len(buf) < self.CHUNK_SIZE:
break
return six.b("").join(data)
def test_tunnel(self):
with self._connect() as rem:
p = (rem.python["-u"] << self.TUNNEL_PROG).popen()
try:
port = int(p.stdout.readline().strip())
except ValueError:
print(p.communicate())
raise
with rem.tunnel(12222, port) as tun:
s = socket.socket()
s.connect(("localhost", 12222))
s.send(six.b("world"))
data = s.recv(100)
s.close()
self.assertEqual(data, six.b("hello world"))
p.communicate()
def test_read_write(self):
with self._connect() as rem:
with rem.tempdir() as dir:
self.assertTrue(dir.isdir())
data = six.b("hello world")
(dir / "foo.txt").write(data)
self.assertEqual((dir / "foo.txt").read(), data)
self.assertFalse(dir.exists())
def test_read_write(self):
with self._connect() as rem:
with rem.tempdir() as dir:
assert dir.is_dir()
data = six.b("hello world")
(dir / "foo.txt").write(data)
assert (dir / "foo.txt").read() == data
assert not dir.exists()
def test_chown(self):
with self._connect() as rem:
with rem.tempdir() as dir:
p = dir / "foo.txt"
p.write(six.b("hello"))
# because we're connected to localhost, we expect UID and GID to be the same
self.assertEqual(p.uid, os.getuid())
self.assertEqual(p.gid, os.getgid())
p.chown(p.uid.name)
self.assertEqual(p.uid, os.getuid())
def run_proc(proc, retcode, timeout = None):
"""Waits for the given process to terminate, with the expected exit code
:param proc: a running Popen-like object
:param retcode: the expected return (exit) code of the process. It defaults to 0 (the
convention for success). If ``None``, the return code is ignored.
It may also be a tuple (or any object that supports ``__contains__``)
of expected return codes.
:param timeout: the number of seconds (a ``float``) to allow the process to run, before
forcefully terminating it. If ``None``, not timeout is imposed; otherwise
the process is expected to terminate within that timeout value, or it will
be killed and :class:`ProcessTimedOut <plumbum.cli.ProcessTimedOut>`
will be raised
:returns: A tuple of (return code, stdout, stderr)
"""
if timeout is not None:
_timeout_queue.put((proc, time.time() + timeout))
stdout, stderr = proc.communicate()
proc._end_time = time.time()
if not stdout:
stdout = six.b("")
if not stderr:
stderr = six.b("")
if getattr(proc, "encoding", None):
stdout = stdout.decode(proc.encoding, "ignore")
stderr = stderr.decode(proc.encoding, "ignore")
if getattr(proc, "_timed_out", False):
raise ProcessTimedOut("Process did not terminate within %s seconds" % (timeout,),
getattr(proc, "argv", None))
if retcode is not None:
if hasattr(retcode, "__contains__"):
if proc.returncode not in retcode:
raise ProcessExecutionError(getattr(proc, "argv", None), proc.returncode,
stdout, stderr)
elif proc.returncode != retcode:
raise ProcessExecutionError(getattr(proc, "argv", None), proc.returncode,
stdout, stderr)
return proc.returncode, stdout, stderr
def test_tempdir(self):
from plumbum.cmd import cat
with local.tempdir() as dir:
self.assertTrue(dir.isdir())
data = six.b("hello world")
with open(str(dir / "test.txt"), "wb") as f:
f.write(data)
with open(str(dir / "test.txt"), "rb") as f:
self.assertEqual(f.read(), data)
self.assertFalse(dir.exists())
def test_tempdir(self):
from plumbum.cmd import cat
with local.tempdir() as dir:
self.assertTrue(dir.isdir())
data = six.b("hello world")
with open(str(dir / "test.txt"), "wb") as f:
f.write(data)
with open(str(dir / "test.txt"), "rb") as f:
self.assertEqual(f.read(), data)
self.assertFalse(dir.exists())
def test_direct_open_tmpdir(self):
from plumbum.cmd import cat
with local.tempdir() as dir:
assert dir.is_dir()
data = six.b("hello world")
with open(dir / "test.txt", "wb") as f:
f.write(data)
with open(dir / "test.txt", "rb") as f:
assert f.read() == data
assert not dir.exists()
def test_tempdir(self):
from plumbum.cmd import cat
with local.tempdir() as dir:
assert dir.is_dir()
data = six.b("hello world")
with open(str(dir / "test.txt"), "wb") as f:
f.write(data)
with open(str(dir / "test.txt"), "rb") as f:
assert f.read() == data
assert not dir.exists()
def test_links(self):
with local.tempdir() as tmp:
src = tmp / "foo.txt"
dst1 = tmp / "bar.txt"
dst2 = tmp / "spam.txt"
data = six.b("hello world")
src.write(data)
src.link(dst1)
self.assertEqual(data, dst1.read())
src.symlink(dst2)
self.assertEqual(data, dst2.read())
def test_links(self):
with local.tempdir() as tmp:
src = tmp / "foo.txt"
dst1 = tmp / "bar.txt"
dst2 = tmp / "spam.txt"
data = six.b("hello world")
src.write(data)
src.link(dst1)
self.assertEqual(data, dst1.read())
src.symlink(dst2)
self.assertEqual(data, dst2.read())
def communicate(self, input = None):
"""Consumes the process' stdout and stderr until the it terminates.
:param input: An optional bytes/buffer object to send to the process over stdin
:returns: A tuple of (stdout, stderr)
"""
stdout = []
stderr = []
sources = [("1", stdout, self.stdout)]
if not self.isatty:
# in tty mode, stdout and stderr are unified
sources.append(("2", stderr, self.stderr))
i = 0
while sources:
if input:
chunk = input[:1000]
self.stdin.write(chunk)
self.stdin.flush()
input = input[1000:]
i = (i + 1) % len(sources)
name, coll, pipe = sources[i]
line = pipe.readline()
shell_logger.debug("%s> %r", name, line)
if not line:
del sources[i]
else:
coll.append(line)
if self.isatty:
stdout.pop(0) # discard first line of prompt
try:
self.returncode = int(stdout.pop(-1))
except (IndexError, ValueError):
self.returncode = "Unknown"
self._done = True
stdout = six.b("").join(stdout)
stderr = six.b("").join(stderr)
return stdout, stderr
def communicate(self):
stdout = []
stderr = []
infile = self.stdin_file
sources = [("1", stdout, self.stdout, self.stdout_file),
("2", stderr, self.stderr, self.stderr_file)]
i = 0
while sources:
if infile:
try:
line = infile.readline()
except (ValueError, IOError):
line = None
logger.debug("communicate: %r", line)
if not line:
infile.close()
infile = None
self.stdin.close()
else:
self.stdin.write(line)
self.stdin.flush()
i = (i + 1) % len(sources)
name, coll, pipe, outfile = sources[i]
line = pipe.readline()
# logger.debug("%s> %r", name, line)
if not line:
del sources[i]
elif outfile:
outfile.write(line)
outfile.flush()
else:
coll.append(line)
self.wait()
stdout = six.b("").join(six.b(s) for s in stdout)
stderr = six.b("").join(six.b(s) for s in stderr)
return stdout, stderr
def communicate(self, input=None):
"""Consumes the process' stdout and stderr until the it terminates.
:param input: An optional bytes/buffer object to send to the process over stdin
:returns: A tuple of (stdout, stderr)
"""
stdout = []
stderr = []
sources = [("1", stdout, self.stdout)]
if not self.isatty:
# in tty mode, stdout and stderr are unified
sources.append(("2", stderr, self.stderr))
i = 0
while sources:
if input:
chunk = input[:1000]
self.stdin.write(chunk)
self.stdin.flush()
input = input[1000:]
i = (i + 1) % len(sources)
name, coll, pipe = sources[i]
line = pipe.readline()
shell_logger.debug("%s> %r", name, line)
if not line:
del sources[i]
else:
coll.append(line)
if self.isatty:
stdout.pop(0) # discard first line of prompt
try:
self.returncode = int(stdout.pop(-1))
except (IndexError, ValueError):
self.returncode = "Unknown"
self._done = True
stdout = six.b("").join(stdout)
stderr = six.b("").join(stderr)
return stdout, stderr
def communicate(self):
stdout = []
stderr = []
infile = self.stdin_file
sources = [("1", stdout, self.stdout, self.stdout_file), ("2", stderr, self.stderr, self.stderr_file)]
i = 0
while sources:
if infile:
try:
line = infile.readline()
except (ValueError, IOError):
line = None
logger.debug("communicate: %r", line)
if not line:
infile.close()
infile = None
self.stdin.close()
else:
self.stdin.write(line)
self.stdin.flush()
i = (i + 1) % len(sources)
name, coll, pipe, outfile = sources[i]
line = pipe.readline()
# logger.debug("%s> %r", name, line)
if not line:
del sources[i]
elif outfile:
outfile.write(line)
outfile.flush()
else:
coll.append(line)
self.wait()
stdout = six.b("").join(six.b(s) for s in stdout)
stderr = six.b("").join(six.b(s) for s in stderr)
return stdout, stderr
def test_path_open_remote_write_local_read(self):
with self._connect() as rem:
# TODO: once Python 2.6 support is dropped, the nested
# with-statements below can be combined using "with x as a, y as b"
with rem.tempdir() as remote_tmpdir:
with local.tempdir() as tmpdir:
assert remote_tmpdir.is_dir()
assert tmpdir.is_dir()
data = six.b("hello world")
with (remote_tmpdir / "bar.txt").open("wb") as f:
f.write(data)
rem.download((remote_tmpdir / "bar.txt"),
(tmpdir / "bar.txt"))
assert (tmpdir / "bar.txt").open("rb").read() == data
assert not remote_tmpdir.exists()
assert not tmpdir.exists()
def test_path_open_local_write_remote_read(self):
with self._connect() as rem:
# TODO: cf. note on Python 2.6 support above
with rem.tempdir() as remote_tmpdir:
with local.tempdir() as tmpdir:
assert remote_tmpdir.is_dir()
assert tmpdir.is_dir()
data = six.b("hello world")
with (tmpdir / "bar.txt").open("wb") as f:
f.write(data)
rem.upload((tmpdir / "bar.txt"),
(remote_tmpdir / "bar.txt"))
assert (remote_tmpdir /
"bar.txt").open("rb").read() == data
assert not remote_tmpdir.exists()
assert not tmpdir.exists()
def test_tunnel(self):
with self._connect() as rem:
p = (rem.python["-u"] << self.TUNNEL_PROG).popen()
try:
port = int(p.stdout.readline().decode("ascii").strip())
except ValueError:
print(p.communicate())
raise
with rem.tunnel(12222, port) as tun:
s = socket.socket()
s.connect(("localhost", 12222))
s.send(six.b("world"))
data = s.recv(100)
s.close()
print(p.communicate())
assert data == b"hello world"
def close(self):
"""Closes (terminates) the shell session"""
if not self.alive():
return
try:
self.proc.stdin.write(six.b("\nexit\n\n\nexit\n\n"))
self.proc.stdin.flush()
time.sleep(0.05)
except (ValueError, EnvironmentError):
pass
for p in [self.proc.stdin, self.proc.stdout, self.proc.stderr]:
try:
p.close()
except Exception:
pass
try:
self.proc.kill()
except EnvironmentError:
pass
self.proc = None
def close(self):
"""Closes (terminates) the shell session"""
if not self.alive():
return
try:
self.proc.stdin.write(six.b("\nexit\n\n\nexit\n\n"))
self.proc.stdin.flush()
time.sleep(0.05)
except (ValueError, EnvironmentError):
pass
for p in [self.proc.stdin, self.proc.stdout, self.proc.stderr]:
try:
p.close()
except Exception:
pass
try:
self.proc.kill()
except EnvironmentError:
pass
self.proc = None
def popen(self, cmd):
"""Runs the given command in the shell, adding some decoration around it. Only a single
command can be executed at any given time.
:param cmd: The command (string or :class:`Command <plumbum.commands.BaseCommand>` object)
to run
:returns: A :class:`SessionPopen <plumbum.session.SessionPopen>` instance
"""
if self.proc is None:
raise ShellSessionError("Shell session has already been closed")
if self._current and not self._current._done:
raise ShellSessionError(
"Each shell may start only one process at a time")
if isinstance(cmd, BaseCommand):
full_cmd = cmd.formulate(1)
else:
full_cmd = cmd
marker = "--.END{}.--".format(time.time() * random.random())
if full_cmd.strip():
full_cmd += " ; "
else:
full_cmd = "true ; "
full_cmd += "echo $? ; echo '{}'".format(marker)
if not self.isatty:
full_cmd += " ; echo '{}' 1>&2".format(marker)
if self.custom_encoding:
full_cmd = full_cmd.encode(self.custom_encoding)
shell_logger.debug("Running %r", full_cmd)
self.proc.stdin.write(full_cmd + six.b("\n"))
self.proc.stdin.flush()
self._current = SessionPopen(
self.proc,
full_cmd,
self.isatty,
self.proc.stdin,
MarkedPipe(self.proc.stdout, marker),
MarkedPipe(self.proc.stderr, marker),
self.custom_encoding,
)
return self._current
def test_tunnel(self):
for tunnel_prog in (self.TUNNEL_PROG_AF_INET,
self.TUNNEL_PROG_AF_UNIX):
with self._connect() as rem:
p = (rem.python["-u"] << tunnel_prog).popen()
port_or_socket = p.stdout.readline().decode("ascii").strip()
try:
port_or_socket = int(port_or_socket)
dhost = "localhost"
except ValueError:
dhost = None
with rem.tunnel(12222, port_or_socket, dhost=dhost) as tun:
s = socket.socket()
s.connect(("localhost", 12222))
s.send(six.b("world"))
data = s.recv(100)
s.close()
print(p.communicate())
assert data == b"hello world"
def communicate(self, input=None):
"""Consumes the process' stdout and stderr until the it terminates.
:param input: An optional bytes/buffer object to send to the process over stdin
:returns: A tuple of (stdout, stderr)
"""
stdout = []
stderr = []
sources = [("1", stdout, self.stdout)]
if not self.isatty:
# in tty mode, stdout and stderr are unified
sources.append(("2", stderr, self.stderr))
i = 0
while sources:
if input:
chunk = input[:1000]
self.stdin.write(chunk)
self.stdin.flush()
input = input[1000:]
i = (i + 1) % len(sources)
name, coll, pipe = sources[i]
try:
line = pipe.readline()
shell_logger.debug("%s> %r", name, line)
except EOFError:
shell_logger.debug("%s> Nothing returned.", name)
self.proc.poll()
returncode = self.proc.returncode
stdout = six.b("").join(stdout).decode(self.custom_encoding,
"ignore")
stderr = six.b("").join(stderr).decode(self.custom_encoding,
"ignore")
argv = self.argv.decode(self.custom_encoding,
"ignore").split(";")[:1]
if returncode == 5:
raise IncorrectLogin(
argv,
returncode,
stdout,
stderr,
message="Incorrect username or password provided",
)
elif returncode == 6:
raise HostPublicKeyUnknown(
argv,
returncode,
stdout,
stderr,
message=
"The authenticity of the host can't be established",
)
elif returncode != 0:
raise SSHCommsError(
argv,
returncode,
stdout,
stderr,
message="SSH communication failed",
)
elif name == "2":
raise SSHCommsChannel2Error(
argv,
returncode,
stdout,
stderr,
message=
"No stderr result detected. Does the remote have Bash as the default shell?",
)
else:
raise SSHCommsError(
argv,
returncode,
stdout,
stderr,
message=
"No communication channel detected. Does the remote exist?",
)
if not line:
del sources[i]
else:
coll.append(line)
if self.isatty:
stdout.pop(0) # discard first line of prompt
try:
self.returncode = int(stdout.pop(-1))
except (IndexError, ValueError):
self.returncode = "Unknown"
self._done = True
stdout = six.b("").join(stdout)
stderr = six.b("").join(stderr)
return stdout, stderr
def test_read_write_bin(self):
with local.tempdir() as tmp:
data = six.b("hello world")
(tmp / "foo.txt").write(data)
assert (tmp / "foo.txt").read(mode='rb') == data
def test_read_write_bin(self):
with local.tempdir() as tmp:
data = six.b("hello world")
(tmp / "foo.txt").write(data)
assert (tmp / "foo.txt").read(mode='rb') == data
def test_read_write(self):
with local.tempdir() as tmp:
data = six.b("hello world")
(tmp / "foo.txt").write(data)
self.assertEqual((tmp / "foo.txt").read(), data)
def test_read_write(self):
with local.tempdir() as tmp:
data = six.b("hello world")
(tmp / "foo.txt").write(data)
self.assertEqual((tmp / "foo.txt").read(), data)