def test_single_tunnel_multi_hosts(self):
remote_host = '127.0.0.8'
remote_server = ThreadedOpenSSHServer(listen_ip=remote_host,
port=self.port)
remote_server.start()
remote_server.wait_for_port()
hosts = [remote_host, remote_host, remote_host]
try:
client = ParallelSSHClient(hosts,
port=self.port,
pkey=self.user_key,
proxy_host=self.proxy_host,
proxy_port=self.port,
num_retries=1,
proxy_pkey=self.user_key)
output = client.run_command(self.cmd, stop_on_errors=False)
client.join(output)
for host, host_out in output.items():
_stdout = list(host_out.stdout)
self.assertListEqual(_stdout, [self.resp])
self.assertEqual(len(hosts), len(list(output.keys())))
del client
finally:
remote_server.stop()
remote_server.join()
def test_tunnel_parallel_client_running_fail(self):
hosts = ['127.0.0.11', '127.0.0.12', '127.0.0.13']
servers = [
OpenSSHServer(listen_ip=_host, port=self.port) for _host in hosts
]
for server in servers:
server.start_server()
try:
client = ParallelSSHClient(
hosts,
port=self.port,
pkey=self.user_key,
proxy_host=self.proxy_host,
proxy_pkey=self.user_key,
proxy_port=self.proxy_port,
num_retries=1,
retry_delay=.1,
)
output = client.run_command(self.cmd)
client.join(output)
for server in (servers[1], servers[2]):
server.stop()
server.server_proc.communicate()
client._host_clients[(1, hosts[1])].disconnect()
client._host_clients[(2, hosts[2])].disconnect()
output = client.run_command(self.cmd, stop_on_errors=False)
client.join(output)
self.assertEqual(len(hosts), len(output))
self.assertTrue(output[1].exception is not None)
self.assertTrue(output[2].exception is not None)
self.assertListEqual(list(output[0].stdout), [self.resp])
finally:
for server in servers:
server.stop()
def test_tunnel_remote_host_timeout(self):
remote_host = '127.0.0.18'
proxy_host = '127.0.0.19'
server = ThreadedOpenSSHServer(listen_ip=proxy_host, port=self.port)
remote_server = ThreadedOpenSSHServer(listen_ip=remote_host,
port=self.port)
for _server in (server, remote_server):
_server.start()
_server.wait_for_port()
try:
client = ParallelSSHClient([remote_host],
port=self.port,
pkey=self.user_key,
proxy_host=proxy_host,
proxy_port=self.port,
num_retries=1,
proxy_pkey=self.user_key)
output = client.run_command(self.cmd)
client.join(output)
client._tunnel.cleanup()
for _server in (server, remote_server):
_server.stop()
_server.join()
# Gevent timeout cannot be caught by stop_on_errors
self.assertRaises(GTimeout,
client.run_command,
self.cmd,
greenlet_timeout=1,
stop_on_errors=False)
finally:
for _server in (server, remote_server):
_server.stop()
def test_tunnel_remote_host_timeout(self):
remote_host = '127.0.0.18'
proxy_host = '127.0.0.19'
server = ThreadedOpenSSHServer(listen_ip=proxy_host, port=self.port)
remote_server = ThreadedOpenSSHServer(listen_ip=remote_host, port=self.port)
for _server in (server, remote_server):
_server.start()
_server.wait_for_port()
try:
client = ParallelSSHClient(
[remote_host], port=self.port, pkey=self.user_key,
proxy_host=proxy_host, proxy_port=self.port, num_retries=1,
proxy_pkey=self.user_key)
output = client.run_command(self.cmd)
client.join(output)
client._tunnel.cleanup()
for _server in (server, remote_server):
_server.stop()
_server.join()
# Gevent timeout cannot be caught by stop_on_errors
self.assertRaises(GTimeout, client.run_command, self.cmd,
greenlet_timeout=1, stop_on_errors=False)
finally:
for _server in (server, remote_server):
_server.stop()
def test_tunnel_parallel_client_part_failure(self):
hosts = ['127.0.0.11', '127.0.0.12', '127.0.0.13', '127.0.0.14']
servers = [
OpenSSHServer(listen_ip=_host, port=self.port) for _host in hosts
]
servers[0].start_server()
servers[1].start_server()
try:
client = ParallelSSHClient(
hosts,
port=self.port,
pkey=self.user_key,
proxy_host=self.proxy_host,
proxy_pkey=self.user_key,
proxy_port=self.proxy_port,
num_retries=1,
)
output = client.run_command(self.cmd, stop_on_errors=False)
client.join(output)
self.assertEqual(len(hosts), len(output))
self.assertTrue(output[2].exception is not None)
self.assertTrue(output[3].exception is not None)
self.assertListEqual(list(output[0].stdout), [self.resp])
self.assertListEqual(list(output[1].stdout), [self.resp])
finally:
for server in servers:
server.stop()
def push_docker_image(hosts: Iterable[RemoteHost], local_image,
local_file: str, remote_file: str) -> None:
"""Push a local Docker image to a set of remote hosts.
Does not verify host keys.
:param hosts: Set of remote host to upload the image to.
:param local_image: Instance of a Docker SDK image object corresponding to the image to
distribute.
:param local_file: Path a temporary file the image is dumped to.
:param remote_file: Path to use for a temporary image file on the remote host.
"""
# Save local image to file.
# Running "docker save %s | gzip > %s" %(local_image.id, local_file) on the host might be
# faster, because the Python API seems to write a temporary file and then return it instead
# of streaming the data.
log.info("Writing image %s to file '%s'.", local_image.short_id,
local_file)
with gzip.open(local_file, 'wb') as file:
for chunk in local_image.save(named=True):
file.write(chunk)
host_ips = [str(host.ssh_host) for host in hosts]
host_config = {
str(host.ssh_host): host.get_host_config()
for host in hosts
}
log.info("Copying image %s to %s.", local_image.short_id,
", ".join("%s (%s)" % (h.name, h.ssh_host) for h in hosts))
try:
ssh_client = ParallelSSHClient(host_ips, host_config=host_config)
# Copy image file to remote hosts.
greenlets = ssh_client.scp_send(local_file, remote_file)
# Not ideal: Waits until all hosts have the image before proceeding.
gevent.joinall(greenlets, raise_error=True)
# Load image from file.
output = ssh_client.run_command("gunzip -c %s | docker image load" %
remote_file)
ssh_client.join(output)
for host, host_output in output.items():
buffer = io.StringIO()
for line in host_output.stdout:
print(line, file=buffer)
level = logging.INFO if host_output.exit_code == 0 else logging.WARNING
log.log(level, "%s responds:\n%s", host, buffer.getvalue())
# Delete the image file.
output = ssh_client.run_command("rm %s" % remote_file)
ssh_client.join(output)
except Exception:
log.error("Pushing docker image to remote hosts failed.")
raise
def test_proxy_error(self):
client = ParallelSSHClient([self.proxy_host],
self.port,
pkey=self.user_key,
proxy_host='127.0.0.155',
proxy_port=123,
num_retries=1)
output = client.run_command(self.cmd, stop_on_errors=False)
client.join(output)
self.assertIsInstance(output[0].exception, ProxyError)
def test_tunnel_init_failure(self):
proxy_host = '127.0.0.20'
client = ParallelSSHClient(
[self.host], port=self.port, pkey=self.user_key,
proxy_host=proxy_host, proxy_port=self.port, num_retries=1,
proxy_pkey=self.user_key)
output = client.run_command(self.cmd, stop_on_errors=False)
client.join(output)
exc = output[self.host].exception
self.assertIsInstance(exc, ProxyError)
self.assertIsInstance(exc.args[1], ConnectionErrorException)
def test_tunnel_init_failure(self):
proxy_host = '127.0.0.20'
client = ParallelSSHClient([self.host],
port=self.port,
pkey=self.user_key,
proxy_host=proxy_host,
proxy_port=self.port,
num_retries=1,
proxy_pkey=self.user_key)
output = client.run_command(self.cmd, stop_on_errors=False)
client.join(output)
exc = output[self.host].exception
self.assertIsInstance(exc, ProxyError)
self.assertIsInstance(exc.args[1], ConnectionErrorException)
def test_tunnel_parallel_client(self):
hosts = ['127.0.0.1%s' % (d, ) for d in range(10)]
servers = [
OpenSSHServer(listen_ip=_host, port=self.port) for _host in hosts
]
for server in servers:
server.start_server()
hosts_5 = [hosts[0], hosts[1], hosts[2], hosts[3], hosts[4]]
try:
client = ParallelSSHClient(
hosts_5,
port=self.port,
pkey=self.user_key,
proxy_host=self.proxy_host,
proxy_pkey=self.user_key,
proxy_port=self.proxy_port,
num_retries=1,
)
start = datetime.now()
output = client.run_command(self.cmd)
end = datetime.now()
dt_5 = end - start
client = ParallelSSHClient(
hosts,
port=self.port,
pkey=self.user_key,
proxy_host=self.proxy_host,
proxy_pkey=self.user_key,
proxy_port=self.proxy_port,
num_retries=1,
)
start = datetime.now()
output = client.run_command(self.cmd)
end = datetime.now()
dt_10 = end - start
dt = dt_10.total_seconds() / dt_5.total_seconds()
# self.assertTrue(dt < 2)
client.join(output)
self.assertEqual(len(hosts), len(output))
for i, host_out in enumerate(output):
_stdout = list(host_out.stdout)
self.assertListEqual(_stdout, [self.resp])
self.assertEqual(hosts[i], host_out.host)
finally:
for server in servers:
server.stop()
def test_tunnel(self):
remote_host = '127.0.0.8'
remote_server = OpenSSHServer(listen_ip=remote_host, port=self.port)
remote_server.start_server()
try:
client = ParallelSSHClient(
[remote_host], port=self.port, pkey=self.user_key,
proxy_host=self.proxy_host, proxy_port=self.port, num_retries=1,
proxy_pkey=self.user_key)
output = client.run_command(self.cmd)
client.join(output)
for host, host_out in output.items():
_stdout = list(host_out.stdout)
self.assertListEqual(_stdout, [self.resp])
self.assertEqual(remote_host, list(output.keys())[0])
del client
finally:
remote_server.stop()
def test_single_tunnel_multi_hosts_timeout(self):
remote_host = '127.0.0.8'
remote_server = ThreadedOpenSSHServer(
listen_ip=remote_host, port=self.port)
remote_server.start()
remote_server.wait_for_port()
hosts = [remote_host, remote_host, remote_host]
try:
client = ParallelSSHClient(
hosts, port=self.port, pkey=self.user_key,
proxy_host=self.proxy_host, proxy_port=self.port, num_retries=1,
proxy_pkey=self.user_key,
timeout=.001)
output = client.run_command(self.cmd, stop_on_errors=False)
client.join(output)
for host, host_out in output.items():
self.assertIsInstance(output[host].exception, Timeout)
finally:
remote_server.stop()
remote_server.join()
def test_tunnel(self):
remote_host = '127.0.0.8'
remote_server = OpenSSHServer(listen_ip=remote_host, port=self.port)
remote_server.start_server()
try:
client = ParallelSSHClient([remote_host],
port=self.port,
pkey=self.user_key,
proxy_host=self.proxy_host,
proxy_port=self.port,
num_retries=1,
proxy_pkey=self.user_key)
output = client.run_command(self.cmd)
client.join(output)
for host, host_out in output.items():
_stdout = list(host_out.stdout)
self.assertListEqual(_stdout, [self.resp])
self.assertEqual(remote_host, list(output.keys())[0])
del client
finally:
remote_server.stop()
def test_single_tunnel_multi_hosts(self):
remote_host = '127.0.0.8'
remote_server = ThreadedOpenSSHServer(
listen_ip=remote_host, port=self.port)
remote_server.start()
remote_server.wait_for_port()
hosts = [remote_host, remote_host, remote_host]
try:
client = ParallelSSHClient(
hosts, port=self.port, pkey=self.user_key,
proxy_host=self.proxy_host, proxy_port=self.port, num_retries=1,
proxy_pkey=self.user_key)
output = client.run_command(self.cmd, stop_on_errors=False)
client.join(output)
for host, host_out in output.items():
_stdout = list(host_out.stdout)
self.assertListEqual(_stdout, [self.resp])
self.assertEqual(len(hosts), len(list(output.keys())))
del client
finally:
remote_server.stop()
remote_server.join()
def test_single_tunnel_multi_hosts_timeout(self):
remote_host = '127.0.0.8'
remote_server = ThreadedOpenSSHServer(listen_ip=remote_host,
port=self.port)
remote_server.start()
remote_server.wait_for_port()
hosts = [remote_host, remote_host, remote_host]
try:
client = ParallelSSHClient(hosts,
port=self.port,
pkey=self.user_key,
proxy_host=self.proxy_host,
proxy_port=self.port,
num_retries=1,
proxy_pkey=self.user_key,
timeout=.001)
output = client.run_command(self.cmd, stop_on_errors=False)
client.join(output)
for host, host_out in output.items():
self.assertIsInstance(output[host].exception, Timeout)
finally:
remote_server.stop()
remote_server.join()
def test_tunnel_host_config(self):
hosts = ['127.0.0.11', '127.0.0.12']
servers = [
OpenSSHServer(listen_ip=_host, port=self.port) for _host in hosts
]
for server in servers:
server.start_server()
host_config = [
HostConfig(proxy_host=self.proxy_host,
proxy_port=self.proxy_port,
proxy_pkey=self.user_key),
HostConfig(proxy_host='127.0.0.155', proxy_port=123),
]
client = ParallelSSHClient(hosts,
port=self.port,
pkey=self.user_key,
host_config=host_config,
num_retries=1)
output = client.run_command(self.cmd, stop_on_errors=False)
client.join(output)
self.assertIsInstance(output[1].exception, ProxyError)
stdout = list(output[0].stdout)
self.assertListEqual(stdout, [self.resp])
def run_command(client: ParallelSSHClient, command: str) -> CommandResult:
"""Executes identical command on all hosts attached to client.
Will wait until all hosts complete the command execution or timeout is reached.
Re-raises pssh exceptions.
# TODO Handle more specific exceptions
"""
# stop_on_errors -> allows others hosts to execute when one crashes, combine exceptions
# output is like: (hostname, host_output)
try:
result = client.run_command(command, stop_on_errors=False)
client.join(result)
except pssh.exceptions.Timeout:
log.warning('Command `{}` reached time limit'.format(command))
raise
except pssh.exceptions.ProxyError as e:
log.error('Could not connect to proxy server, reason: {}'.format(e))
raise
except Exception as e:
log.critical(e)
raise # FIXME Find out what throws this exception
else:
log.debug('Command `{}` finished'.format(command))
return result
