def test_create_neither_present(self):
values = {
"which %s" % RemoteFirewallControlFirewallCmd.firewall_app_name:
Shell.RunResult(1, "", "", False),
"which %s" % RemoteFirewallControlIpTables.firewall_app_name:
Shell.RunResult(1, "", "", False),
}
def side_effect(address, cmd):
return values[cmd]
self.mock_ssh.side_effect = side_effect
with self.assertRaises(RuntimeError):
RemoteFirewallControl.create("0.0.0.0", RRO(None)._ssh_address)
def _fake_run(self,
arg_list,
logger=None,
monitor_func=None,
timeout=Shell.SHELLTIMEOUT,
shell=False):
assert type(arg_list) in [
list, str, unicode
], 'arg list must be list or str :%s' % type(arg_list)
# Allow simple commands to just be presented as a string. However do not start formatting the string this
# will be rejected in a code review. If it has args present them as a list.
if type(arg_list) in [str, unicode]:
arg_list = arg_list.split()
args = tuple(arg_list)
self._commands_history.append(args)
try:
result = self._get_executable_command(args)
result.executions_remaining -= 1
if result.rc == self.CommandNotFound:
raise OSError(errno.ENOENT, result.stderr)
return Shell.RunResult(result.rc, result.stdout, result.stderr,
False)
except KeyError:
raise OSError(errno.ENOENT,
self._missing_command_err_msg % ' '.join(arg_list))
def test_create(self):
values = {
"which %s" % RemoteFirewallControlFirewallCmd.firewall_app_name:
Shell.RunResult(1, "", "", False),
"which %s" % RemoteFirewallControlIpTables.firewall_app_name:
Shell.RunResult(0, "", "", False),
}
def side_effect(address, cmd):
return values[cmd]
self.mock_ssh.side_effect = side_effect
new_controller = RemoteFirewallControl.create("0.0.0.0",
RRO(None)._ssh_address)
self.assertEquals(type(new_controller), RemoteFirewallControlIpTables)
def run(arg_list):
values = {
("rpm", "-q", "--whatprovides", "kmod-lustre"):
"kmod-lustre-1.2.3-1.el6.x86_64\n",
("uname", "-r"):
"2.6.32-358.2.1.el6.x86_64\n",
("rpm", "-q", "kernel"):
"kernel-2.6.32-358.2.1.el6.x86_64\n"
"kernel-2.6.32-358.18.1.el6_lustre.x86_64\n"
}
return Shell.RunResult(0, values[tuple(arg_list)], "", False)
def example_func_1(address, command, expected_return_code=None):
# example output from 'iptables -L' or 'service iptables status' if firewall not configured
not_configured_output = """Table: filter
Chain INPUT (policy ACCEPT)
num target prot opt source destination
Chain FORWARD (policy ACCEPT)
num target prot opt source destination
Chain OUTPUT (policy ACCEPT)
num target prot opt source destination
"""
return Shell.RunResult(0, not_configured_output, "", False)
def example_func_3(address, command, expected_return_code=None):
# example output from 'iptables -L' or 'service iptables status'
chain_output = """Table: filter
Chain INPUT (policy ACCEPT)
num target prot opt source destination
1 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 state RELATED,ESTABLISHED
2 ACCEPT icmp -- 0.0.0.0/0 0.0.0.0/0
3 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0
4 ACCEPT udp -- 0.0.0.0/0 0.0.0.0/0 state NEW udp dpt:123
5 ACCEPT tcp -- 0.0.0.0/0 0.0.0.0/0 state NEW tcp dpt:22
6 ACCEPT tcp -- 0.0.0.0/0 0.0.0.0/0 state NEW tcp dpt:80
7 ACCEPT tcp -- 0.0.0.0/0 0.0.0.0/0 state NEW tcp dpt:443
8 REJECT all -- 0.0.0.0/0 0.0.0.0/0 reject-with icmp-host-prohibited
Chain FORWARD (policy ACCEPT)
num target prot opt source destination
1 REJECT all -- 0.0.0.0/0 0.0.0.0/0 reject-with icmp-host-prohibited
Chain OUTPUT (policy ACCEPT)
num target prot opt source destination
"""
return Shell.RunResult(0, chain_output, "", False)
def _ssh_address(self,
address,
command,
expected_return_code=0,
timeout=TEST_TIMEOUT,
buffer=None,
as_root=True):
"""
Executes a command on a remote server over ssh.
Sends a command over ssh to a remote machine and returns the stdout,
stderr, and exit status. It will verify that the exit status of the
command matches expected_return_code unless expected_return_code=None.
"""
def host_test(address, issue_num):
def print_result(r):
return "rc: %s\n\nstdout:\n%s\n\nstderr:\n%s" % (
r.rc, r.stdout, r.stderr)
ping_result1 = Shell.run(["ping", "-c", "1", "-W", "1", address])
ping_result2_report = ""
ip_addr_result = Shell.run(["ip", "addr", "ls"])
ip_route_ls_result = Shell.run(["ip", "route", "ls"])
try:
gw = [
l for l in ip_route_ls_result.stdout.split("\n")
if l.startswith("default ")
][0].split()[2]
ping_gw_result = Shell.run(["ping", "-c", "1", "-W", "1", gw])
ping_gw_report = "\nping gateway (%s): %s" % (
gw, print_result(ping_gw_result))
except:
ping_gw_report = ("\nUnable to ping gatewy. "
"No gateway could be found in:\n" %
ip_route_ls_result.stdout)
if ping_result1.rc != 0:
time.sleep(30)
ping_result2 = Shell.run(
["ping", "-c", "1", "-W", "1", address])
ping_result2_report = "\n30s later ping: %s" % print_result(
ping_result2)
msg = (
"Error connecting to %s: %s.\n"
"Please add the following to "
"https://github.com/whamcloud/integrated-manager-for-lustre/issues/%s\n"
"Performing some diagnostics...\n"
"ping: %s\n"
"ifconfig -a: %s\n"
"ip route ls: %s"
"%s"
"%s" % (
address,
e,
issue_num,
print_result(ping_result1),
print_result(ip_addr_result),
print_result(ip_route_ls_result),
ping_gw_report,
ping_result2_report,
))
logger.error(msg)
DEVNULL = open(os.devnull, "wb")
p = subprocess.Popen(["sendmail", "-t"],
stdin=subprocess.PIPE,
stdout=DEVNULL,
stderr=DEVNULL)
p.communicate(input=b"To: [email protected]\n"
b"Subject: GH#%s\n\n" % issue_num + msg)
p.wait()
DEVNULL.close()
logger.debug("remote_command[%s]: %s" % (address, command))
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# the set -e just sets up a fail-safe execution environment where
# any shell commands in command that fail and are not error checked
# cause the shell to fail, alerting the caller that one of their
# commands failed unexpectedly
command = "set -e; %s" % command
# exec 0<&- being prefixed to the shell command string below closes
# the shell's stdin as we don't expect any uses of remote_command()
# to read from stdin
if not buffer:
command = "exec 0<&-; %s" % command
args = {"username": "******"}
# If given an ssh_config file, require that it defines
# a private key and username for accessing this host
config_path = config.get("ssh_config", None)
if config_path:
ssh_config = paramiko.SSHConfig()
ssh_config.parse(open(config_path))
host_config = ssh_config.lookup(address)
address = host_config["hostname"]
if "user" in host_config:
args["username"] = host_config["user"]
if args["username"] != "root" and as_root:
command = 'sudo sh -c "{}"'.format(
command.replace('"', '\\"'))
if "identityfile" in host_config:
args["key_filename"] = host_config["identityfile"][0]
# Work around paramiko issue 157, failure to parse quoted values
# (vagrant always quotes IdentityFile)
args["key_filename"] = args["key_filename"].strip('"')
logger.info(
"SSH address = %s, timeout = %d, write len = %d, args = %s" %
(address, timeout, len(buffer or ""), args))
# Create ssh connection
try:
ssh.connect(address, **args)
except paramiko.ssh_exception.SSHException as e:
host_test(address, "29")
return Shell.RunResult(1, "", "", timeout=False)
transport = ssh.get_transport()
transport.set_keepalive(20)
channel = transport.open_session()
channel.settimeout(timeout)
# Actually execute the command
try:
channel.exec_command(command)
except paramiko.transport.Socket as e:
host_test(address, "72")
return Shell.RunResult(1, "", "", timeout=False)
if buffer:
stdin = channel.makefile("wb")
stdin.write(buffer)
stdin.close()
# Always shutdown write to ensure executable does not wait on input
channel.shutdown_write()
# Store results. This needs to happen in this order. If recv_exit_status is
# read first, it can block indefinitely due to paramiko bug #448. The read on
# the stdout will wait until the command finishes, so its not necessary to have
# recv_exit_status to block on it first. Closing the channel must come last,
# or else reading from stdout/stderr will return an empty string.
stdout = channel.makefile("rb").read()
stderr = channel.makefile_stderr("rb").read()
rc = channel.recv_exit_status()
channel.close()
# Verify we recieved the correct exit status if one was specified.
if expected_return_code is not None:
self._test_case.assertEqual(
rc,
expected_return_code,
"rc (%s) != expected_return_code (%s), stdout: '%s', stderr: '%s'"
% (rc, expected_return_code, stdout, stderr),
)
return Shell.RunResult(rc, stdout, stderr, timeout=False)
def example_func_2(address, command, expected_return_code=None):
return Shell.RunResult(0, "", "", False)
def example_func_4(address, command, expected_return_code=None):
list_ports_output = """123/udp 22/tcp 80/tcp 443/tcp
"""
return Shell.RunResult(0, list_ports_output, "", False)
def example_func_2(address, command, expected_return_code=None):
return Shell.RunResult(0, "Chain INPUT (policy ACCEPT)", "", False)
def run(arg_list):
values = {("getenforce", ): ""}
return Shell.RunResult(127, values[tuple(arg_list)],
"getenforce: command not found", False)
def run(arg_list):
values = {("getenforce", ): "Enforcing\n"}
return Shell.RunResult(0, values[tuple(arg_list)], "", False)
def run(arg_list):
values = {
("rpm", "-q", "kernel"): {
"rc":
0,
"stdout":
"kernel-2.6.32-358.2.1.el6.x86_64\n"
"kernel-2.6.32-358.18.1.el6_lustre.x86_64\n",
},
("rpm", "-q", "--whatprovides", "kmod-lustre"): {
"rc": 0,
"stdout": "kmod-lustre-1.2.3-1.el6.x86_64\n",
},
("rpm", "-ql", "--whatprovides", "lustre-osd", "kmod-lustre"):
{
"rc":
0,
"stdout":
"/lib/modules/2.6.32-358.18.1.el7_lustre.x86_64/extra/lustre/fs/lustre.ko\n"
"/lib/modules/2.6.32-358.18.1.el7_lustre.x86_64/extra/lustre-osd-ldiskfs/fs/osd_ldiskfs.ko\n",
},
(
"modinfo",
"-n",
"-k",
"2.6.32-358.2.1.el6.x86_64",
"lustre",
"osd_ldiskfs",
): {
"rc": 1,
"stdout": ""
},
(
"modinfo",
"-n",
"-k",
"2.6.32-358.18.1.el6_lustre.x86_64",
"lustre",
"osd_ldiskfs",
): {
"rc":
0,
"stdout":
"/lib/modules/2.6.32-358.18.1.el7_lustre.x86_64/extra/lustre/fs/lustre.ko\n"
"/lib/modules/2.6.32-358.18.1.el7_lustre.x86_64/extra/lustre-osd-ldiskfs/fs/osd_ldiskfs.ko\n",
},
("uname", "-m"): {
"rc": 0,
"stdout": "x86_64\n"
},
("uname", "-r"): {
"rc": 0,
"stdout": "2.6.32-358.2.1.el6.x86_64\n"
},
}
return Shell.RunResult(
values[tuple(arg_list)]["rc"],
values[tuple(arg_list)]["stdout"],
"",
False,
)
def _ssh_address(self,
address,
command,
expected_return_code=0,
timeout=TEST_TIMEOUT,
buffer=None,
as_root=True):
"""
Executes a command on a remote server over ssh.
Sends a command over ssh to a remote machine and returns the stdout,
stderr, and exit status. It will verify that the exit status of the
command matches expected_return_code unless expected_return_code=None.
"""
def host_test(address, issue_num):
def print_result(r):
return "rc: %s\n\nstdout:\n%s\n\nstderr:\n%s" % \
(r.rc, r.stdout, r.stderr)
ping_result1 = Shell.run(['ping', '-c', '1', '-W', '1', address])
ping_result2_report = ""
ip_addr_result = Shell.run(['ip', 'addr', 'ls'])
ip_route_ls_result = Shell.run(['ip', 'route', 'ls'])
try:
gw = [
l for l in ip_route_ls_result.stdout.split('\n')
if l.startswith("default ")
][0].split()[2]
ping_gw_result = Shell.run(['ping', '-c', '1', '-W', '1', gw])
ping_gw_report = "\nping gateway (%s): %s" % \
(gw, print_result(ping_gw_result))
except:
ping_gw_report = "\nUnable to ping gatewy. " \
"No gateway could be found in:\n" % \
ip_route_ls_result.stdout
if ping_result1.rc != 0:
time.sleep(30)
ping_result2 = Shell.run(
['ping', '-c', '1', '-W', '1', address])
ping_result2_report = "\n30s later ping: %s" % \
print_result(ping_result2)
msg = "Error connecting to %s: %s.\n" \
"Please add the following to " \
"https://github.com/intel-hpdd/intel-manager-for-lustre/issues/%s\n" \
"Performing some diagnostics...\n" \
"ping: %s\n" \
"ifconfig -a: %s\n" \
"ip route ls: %s" \
"%s" \
"%s" % \
(address, e,
issue_num,
print_result(ping_result1),
print_result(ip_addr_result),
print_result(ip_route_ls_result),
ping_gw_report,
ping_result2_report)
logger.error(msg)
DEVNULL = open(os.devnull, 'wb')
p = subprocess.Popen(['sendmail', '-t'],
stdin=subprocess.PIPE,
stdout=DEVNULL,
stderr=DEVNULL)
p.communicate(input=b'To: [email protected]\n'
'Subject: GH#%s\n\n' % issue_num + msg)
p.wait()
DEVNULL.close()
logger.debug("remote_command[%s]: %s" % (address, command))
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# the set -e just sets up a fail-safe execution environment where
# any shell commands in command that fail and are not error checked
# cause the shell to fail, alerting the caller that one of their
# commands failed unexpectedly
command = "set -e; %s" % command
# exec 0<&- being prefixed to the shell command string below closes
# the shell's stdin as we don't expect any uses of remote_command()
# to read from stdin
if not buffer:
command = "exec 0<&-; %s" % command
args = {'username': '******'}
# If given an ssh_config file, require that it defines
# a private key and username for accessing this host
config_path = config.get('ssh_config', None)
if config_path:
ssh_config = paramiko.SSHConfig()
ssh_config.parse(open(config_path))
host_config = ssh_config.lookup(address)
address = host_config['hostname']
if 'user' in host_config:
args['username'] = host_config['user']
if args['username'] != 'root' and as_root:
command = "sudo sh -c \"{}\"".format(
command.replace('"', '\\"'))
if 'identityfile' in host_config:
args['key_filename'] = host_config['identityfile'][0]
# Work around paramiko issue 157, failure to parse quoted values
# (vagrant always quotes IdentityFile)
args['key_filename'] = args['key_filename'].strip("\"")
logger.info("SSH address = %s, args = %s" % (address, args))
# Create ssh connection
try:
ssh.connect(address, **args)
except paramiko.ssh_exception.SSHException, e:
host_test(address, "29")
return Shell.RunResult(1, "", "", timeout=False)
class RealRemoteOperations(RemoteOperations):
def __init__(self, test_case):
self._test_case = test_case
def fail_connections(self, fail):
# Ways to implement this outside simulation:
# * Insert a firewall rule to drop packages between agent and manager
# * Stop the management network interface on the storage server
# * Switch off the management switch port that the storage server is connected to
raise NotImplementedError()
def drop_responses(self, fail):
# Ways to implement this outside simulation:
# * Insert a transparent HTTP proxy between agent and manager, which drops responses
# * Use a firewall rule to drop manager->agent TCP streams after N bytes to cause responses
# to be mangled.
raise NotImplementedError()
def _ssh_address_no_check(self, address, command_string):
""" pass _ssh_address expected_return_code=None, so exception not raised on nonzero return code """
return self._ssh_address(address,
command_string,
expected_return_code=None)
# TODO: reconcile this with the one in UtilityTestCase, ideally all remote
# operations would flow through here to avoid rogue SSH calls
def _ssh_address(self,
address,
command,
expected_return_code=0,
timeout=TEST_TIMEOUT,
buffer=None,
as_root=True):
"""
Executes a command on a remote server over ssh.
Sends a command over ssh to a remote machine and returns the stdout,
stderr, and exit status. It will verify that the exit status of the
command matches expected_return_code unless expected_return_code=None.
"""
def host_test(address, issue_num):
def print_result(r):
return "rc: %s\n\nstdout:\n%s\n\nstderr:\n%s" % \
(r.rc, r.stdout, r.stderr)
ping_result1 = Shell.run(['ping', '-c', '1', '-W', '1', address])
ping_result2_report = ""
ip_addr_result = Shell.run(['ip', 'addr', 'ls'])
ip_route_ls_result = Shell.run(['ip', 'route', 'ls'])
try:
gw = [
l for l in ip_route_ls_result.stdout.split('\n')
if l.startswith("default ")
][0].split()[2]
ping_gw_result = Shell.run(['ping', '-c', '1', '-W', '1', gw])
ping_gw_report = "\nping gateway (%s): %s" % \
(gw, print_result(ping_gw_result))
except:
ping_gw_report = "\nUnable to ping gatewy. " \
"No gateway could be found in:\n" % \
ip_route_ls_result.stdout
if ping_result1.rc != 0:
time.sleep(30)
ping_result2 = Shell.run(
['ping', '-c', '1', '-W', '1', address])
ping_result2_report = "\n30s later ping: %s" % \
print_result(ping_result2)
msg = "Error connecting to %s: %s.\n" \
"Please add the following to " \
"https://github.com/intel-hpdd/intel-manager-for-lustre/issues/%s\n" \
"Performing some diagnostics...\n" \
"ping: %s\n" \
"ifconfig -a: %s\n" \
"ip route ls: %s" \
"%s" \
"%s" % \
(address, e,
issue_num,
print_result(ping_result1),
print_result(ip_addr_result),
print_result(ip_route_ls_result),
ping_gw_report,
ping_result2_report)
logger.error(msg)
DEVNULL = open(os.devnull, 'wb')
p = subprocess.Popen(['sendmail', '-t'],
stdin=subprocess.PIPE,
stdout=DEVNULL,
stderr=DEVNULL)
p.communicate(input=b'To: [email protected]\n'
'Subject: GH#%s\n\n' % issue_num + msg)
p.wait()
DEVNULL.close()
logger.debug("remote_command[%s]: %s" % (address, command))
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# the set -e just sets up a fail-safe execution environment where
# any shell commands in command that fail and are not error checked
# cause the shell to fail, alerting the caller that one of their
# commands failed unexpectedly
command = "set -e; %s" % command
# exec 0<&- being prefixed to the shell command string below closes
# the shell's stdin as we don't expect any uses of remote_command()
# to read from stdin
if not buffer:
command = "exec 0<&-; %s" % command
args = {'username': '******'}
# If given an ssh_config file, require that it defines
# a private key and username for accessing this host
config_path = config.get('ssh_config', None)
if config_path:
ssh_config = paramiko.SSHConfig()
ssh_config.parse(open(config_path))
host_config = ssh_config.lookup(address)
address = host_config['hostname']
if 'user' in host_config:
args['username'] = host_config['user']
if args['username'] != 'root' and as_root:
command = "sudo sh -c \"{}\"".format(
command.replace('"', '\\"'))
if 'identityfile' in host_config:
args['key_filename'] = host_config['identityfile'][0]
# Work around paramiko issue 157, failure to parse quoted values
# (vagrant always quotes IdentityFile)
args['key_filename'] = args['key_filename'].strip("\"")
logger.info("SSH address = %s, args = %s" % (address, args))
# Create ssh connection
try:
ssh.connect(address, **args)
except paramiko.ssh_exception.SSHException, e:
host_test(address, "29")
return Shell.RunResult(1, "", "", timeout=False)
transport = ssh.get_transport()
transport.set_keepalive(20)
channel = transport.open_session()
channel.settimeout(timeout)
# Actually execute the command
try:
channel.exec_command(command)
except paramiko.transport.Socket, e:
host_test(address, "72")
return Shell.RunResult(1, "", "", timeout=False)
# the stdout will wait until the command finishes, so its not necessary to have
# recv_exit_status to block on it first. Closing the channel must come last,
# or else reading from stdout/stderr will return an empty string.
stdout = channel.makefile('rb').read()
stderr = channel.makefile_stderr('rb').read()
rc = channel.recv_exit_status()
channel.close()
# Verify we recieved the correct exit status if one was specified.
if expected_return_code is not None:
self._test_case.assertEqual(
rc, expected_return_code,
"rc (%s) != expected_return_code (%s), stdout: '%s', stderr: '%s'"
% (rc, expected_return_code, stdout, stderr))
return Shell.RunResult(rc, stdout, stderr, timeout=False)
def _ssh_fqdn(self,
fqdn,
command,
expected_return_code=0,
timeout=TEST_TIMEOUT,
buffer=None):
address = None
for host in config['lustre_servers']:
if host['fqdn'] == fqdn:
address = host['address']
if address is None:
raise KeyError(fqdn)
return self._ssh_address(address, command, expected_return_code,