def process_rules(self):
        """
        Process string from firewall-cmd application output on EL7, matches ANY source/dest address rule specifications

        Note: the expected input is line separated <port>/<proto> pairs, this will give any explicitly allowed rules
        added by IML and other applications to the default zone but will not list named services enabled in firewalld
        Empty string is a valid input indicating no explicitly added port/proto rules

        :return: None on success/valid input, error string otherwise
        """
        result = self.remote_access_func(self.address, self.firewall_list_cmd)

        if result.rc != 0:
            from iml_common.lib.shell import Shell

            raise RuntimeError(
                """process_rules(): remote shell command failed unexpectedly (%s), is firewall-cmd running? (%s) (%s)
systemctl status firewalld:
%s

systemctl status polkit:
%s

journalctl -n 100:
%s"""
                % (
                    result.rc,
                    result.stdout,
                    result.stderr,
                    Shell.run(["systemctl", "status", "firewalld"]).stdout,
                    Shell.run(["systemctl", "status", "polkit"]).stdout,
                    Shell.run(["journalctl", "-n", "100"]).stdout,
                )
            )

        if result.stdout.strip() == "":
            return None

        # handle output separated by either new-line chars, spaces, or both
        tokens = [token for token in result.stdout.replace("\n", " ").split() if token]

        # as we are reading in a new firewall table content, reset 'rules' member list
        self.rules = []
        index = 0

        while index < len(tokens):
            match = re.search("(?P<port>\d+)\/(?P<proto>\w+)", tokens[index])

            if match:
                # Note: because we are uncertain about the command used to obtain the input string, assume persist=False
                self.rules.append(self.firewall_rule(match.group("port"), match.group("proto")))
            else:
                raise RuntimeError(
                    'process_rules(): "%s" command output contains unexpected firewall-cmd output'
                    % self.firewall_list_cmd
                )

            index += 1

        return None
        def host_test(address, issue_num):
            def print_result(r):
                return "rc: %s\n\nstdout:\n%s\n\nstderr:\n%s" % \
                       (r.rc, r.stdout, r.stderr)

            ping_result1 = Shell.run(['ping', '-c', '1', '-W', '1', address])
            ping_result2_report = ""
            ip_addr_result = Shell.run(['ip', 'addr', 'ls'])
            ip_route_ls_result = Shell.run(['ip', 'route', 'ls'])

            try:
                gw = [
                    l for l in ip_route_ls_result.stdout.split('\n')
                    if l.startswith("default ")
                ][0].split()[2]
                ping_gw_result = Shell.run(['ping', '-c', '1', '-W', '1', gw])
                ping_gw_report = "\nping gateway (%s): %s" % \
                    (gw, print_result(ping_gw_result))
            except:
                ping_gw_report = "\nUnable to ping gatewy.  " \
                                 "No gateway could be found in:\n" % \
                                 ip_route_ls_result.stdout

            if ping_result1.rc != 0:
                time.sleep(30)
                ping_result2 = Shell.run(
                    ['ping', '-c', '1', '-W', '1', address])
                ping_result2_report = "\n30s later ping: %s" % \
                    print_result(ping_result2)

            msg = "Error connecting to %s: %s.\n" \
                  "Please add the following to " \
                  "https://github.com/intel-hpdd/intel-manager-for-lustre/issues/%s\n" \
                  "Performing some diagnostics...\n" \
                  "ping: %s\n" \
                  "ifconfig -a: %s\n" \
                  "ip route ls: %s" \
                  "%s" \
                  "%s" % \
                  (address, e,
                   issue_num,
                   print_result(ping_result1),
                   print_result(ip_addr_result),
                   print_result(ip_route_ls_result),
                   ping_gw_report,
                   ping_result2_report)

            logger.error(msg)

            DEVNULL = open(os.devnull, 'wb')
            p = subprocess.Popen(['sendmail', '-t'],
                                 stdin=subprocess.PIPE,
                                 stdout=DEVNULL,
                                 stderr=DEVNULL)
            p.communicate(input=b'To: [email protected]\n'
                          'Subject: GH#%s\n\n' % issue_num + msg)
            p.wait()
            DEVNULL.close()
        def host_test(address, issue_num):
            def print_result(r):
                return "rc: %s\n\nstdout:\n%s\n\nstderr:\n%s" % (r.rc, r.stdout, r.stderr)

            ping_result1 = Shell.run(["ping", "-c", "1", "-W", "1", address])
            ping_result2_report = ""
            ip_addr_result = Shell.run(["ip", "addr", "ls"])
            ip_route_ls_result = Shell.run(["ip", "route", "ls"])

            try:
                gw = [l for l in ip_route_ls_result.stdout.split("\n") if l.startswith("default ")][0].split()[2]
                ping_gw_result = Shell.run(["ping", "-c", "1", "-W", "1", gw])
                ping_gw_report = "\nping gateway (%s): %s" % (gw, print_result(ping_gw_result))
            except:
                ping_gw_report = (
                    "\nUnable to ping gatewy.  " "No gateway could be found in:\n" % ip_route_ls_result.stdout
                )

            if ping_result1.rc != 0:
                time.sleep(30)
                ping_result2 = Shell.run(["ping", "-c", "1", "-W", "1", address])
                ping_result2_report = "\n30s later ping: %s" % print_result(ping_result2)

            msg = (
                "Error connecting to %s: %s.\n"
                "Please add the following to "
                "https://github.com/whamcloud/integrated-manager-for-lustre/issues/%s\n"
                "Performing some diagnostics...\n"
                "ping: %s\n"
                "ifconfig -a: %s\n"
                "ip route ls: %s"
                "%s"
                "%s"
                % (
                    address,
                    e,
                    issue_num,
                    print_result(ping_result1),
                    print_result(ip_addr_result),
                    print_result(ip_route_ls_result),
                    ping_gw_report,
                    ping_result2_report,
                )
            )

            logger.error(msg)

            DEVNULL = open(os.devnull, "wb")
            p = subprocess.Popen(["sendmail", "-t"], stdin=subprocess.PIPE, stdout=DEVNULL, stderr=DEVNULL)
            p.communicate(input=b"To: [email protected]\n" b"Subject: GH#%s\n\n" % issue_num + msg)
            p.wait()
            DEVNULL.close()
    def test_create_neither_present(self):
        values = {
            "which %s" % RemoteFirewallControlFirewallCmd.firewall_app_name:
            Shell.RunResult(1, "", "", False),
            "which %s" % RemoteFirewallControlIpTables.firewall_app_name:
            Shell.RunResult(1, "", "", False),
        }

        def side_effect(address, cmd):
            return values[cmd]

        self.mock_ssh.side_effect = side_effect

        with self.assertRaises(RuntimeError):
            RemoteFirewallControl.create("0.0.0.0", RRO(None)._ssh_address)
    def _fake_run(self,
                  arg_list,
                  logger=None,
                  monitor_func=None,
                  timeout=Shell.SHELLTIMEOUT,
                  shell=False):
        assert type(arg_list) in [
            list, str, unicode
        ], 'arg list must be list or str :%s' % type(arg_list)

        # Allow simple commands to just be presented as a string. However do not start formatting the string this
        # will be rejected in a code review. If it has args present them as a list.
        if type(arg_list) in [str, unicode]:
            arg_list = arg_list.split()

        args = tuple(arg_list)
        self._commands_history.append(args)

        try:
            result = self._get_executable_command(args)
            result.executions_remaining -= 1

            if result.rc == self.CommandNotFound:
                raise OSError(errno.ENOENT, result.stderr)

            return Shell.RunResult(result.rc, result.stdout, result.stderr,
                                   False)
        except KeyError:
            raise OSError(errno.ENOENT,
                          self._missing_command_err_msg % ' '.join(arg_list))
    def test_create(self):
        values = {
            "which %s" % RemoteFirewallControlFirewallCmd.firewall_app_name:
            Shell.RunResult(1, "", "", False),
            "which %s" % RemoteFirewallControlIpTables.firewall_app_name:
            Shell.RunResult(0, "", "", False),
        }

        def side_effect(address, cmd):
            return values[cmd]

        self.mock_ssh.side_effect = side_effect

        new_controller = RemoteFirewallControl.create("0.0.0.0",
                                                      RRO(None)._ssh_address)

        self.assertEquals(type(new_controller), RemoteFirewallControlIpTables)
Esempio n. 7
0
 def run(arg_list):
     values = {
         ("rpm", "-q", "--whatprovides", "kmod-lustre"):
         "kmod-lustre-1.2.3-1.el6.x86_64\n",
         ("uname", "-r"):
         "2.6.32-358.2.1.el6.x86_64\n",
         ("rpm", "-q", "kernel"):
         "kernel-2.6.32-358.2.1.el6.x86_64\n"
         "kernel-2.6.32-358.18.1.el6_lustre.x86_64\n"
     }
     return Shell.RunResult(0, values[tuple(arg_list)], "", False)
    def example_func_1(address, command, expected_return_code=None):
        # example output from 'iptables -L' or 'service iptables status' if firewall not configured
        not_configured_output = """Table: filter
Chain INPUT (policy ACCEPT)
num  target     prot opt source               destination

Chain FORWARD (policy ACCEPT)
num  target     prot opt source               destination

Chain OUTPUT (policy ACCEPT)
num  target     prot opt source               destination

"""
        return Shell.RunResult(0, not_configured_output, "", False)
    def example_func_3(address, command, expected_return_code=None):
        # example output from 'iptables -L' or 'service iptables status'
        chain_output = """Table: filter
Chain INPUT (policy ACCEPT)
num  target     prot opt source               destination
1    ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0           state RELATED,ESTABLISHED
2    ACCEPT     icmp --  0.0.0.0/0            0.0.0.0/0
3    ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0
4    ACCEPT     udp  --  0.0.0.0/0            0.0.0.0/0           state NEW udp dpt:123
5    ACCEPT     tcp  --  0.0.0.0/0            0.0.0.0/0           state NEW tcp dpt:22
6    ACCEPT     tcp  --  0.0.0.0/0            0.0.0.0/0           state NEW tcp dpt:80
7    ACCEPT     tcp  --  0.0.0.0/0            0.0.0.0/0           state NEW tcp dpt:443
8    REJECT     all  --  0.0.0.0/0            0.0.0.0/0           reject-with icmp-host-prohibited

Chain FORWARD (policy ACCEPT)
num  target     prot opt source               destination
1    REJECT     all  --  0.0.0.0/0            0.0.0.0/0           reject-with icmp-host-prohibited

Chain OUTPUT (policy ACCEPT)
num  target     prot opt source               destination

"""
        return Shell.RunResult(0, chain_output, "", False)
 def run(arg_list):
     values = {("getenforce", ): "Enforcing\n"}
     return Shell.RunResult(0, values[tuple(arg_list)], "", False)
 def example_func_2(address, command, expected_return_code=None):
     return Shell.RunResult(0, "", "", False)
        # the stdout will wait until the command finishes, so its not necessary to have
        # recv_exit_status to block on it first. Closing the channel must come last,
        # or else reading from stdout/stderr will return an empty string.
        stdout = channel.makefile('rb').read()
        stderr = channel.makefile_stderr('rb').read()
        rc = channel.recv_exit_status()
        channel.close()

        # Verify we recieved the correct exit status if one was specified.
        if expected_return_code is not None:
            self._test_case.assertEqual(
                rc, expected_return_code,
                "rc (%s) != expected_return_code (%s), stdout: '%s', stderr: '%s'"
                % (rc, expected_return_code, stdout, stderr))

        return Shell.RunResult(rc, stdout, stderr, timeout=False)

    def _ssh_fqdn(self,
                  fqdn,
                  command,
                  expected_return_code=0,
                  timeout=TEST_TIMEOUT,
                  buffer=None):
        address = None
        for host in config['lustre_servers']:
            if host['fqdn'] == fqdn:
                address = host['address']
        if address is None:
            raise KeyError(fqdn)

        return self._ssh_address(address, command, expected_return_code,
    def example_func_4(address, command, expected_return_code=None):
        list_ports_output = """123/udp 22/tcp 80/tcp 443/tcp
"""
        return Shell.RunResult(0, list_ports_output, "", False)
 def example_func_2(address, command, expected_return_code=None):
     return Shell.RunResult(0, "Chain INPUT (policy ACCEPT)", "", False)
class RealRemoteOperations(RemoteOperations):
    def __init__(self, test_case):
        self._test_case = test_case

    def fail_connections(self, fail):
        # Ways to implement this outside simulation:
        #  * Insert a firewall rule to drop packages between agent and manager
        #  * Stop the management network interface on the storage server
        #  * Switch off the management switch port that the storage server is connected to
        raise NotImplementedError()

    def drop_responses(self, fail):
        # Ways to implement this outside simulation:
        #  * Insert a transparent HTTP proxy between agent and manager, which drops responses
        #  * Use a firewall rule to drop manager->agent TCP streams after N bytes to cause responses
        #    to be mangled.
        raise NotImplementedError()

    def _ssh_address_no_check(self, address, command_string):
        """ pass _ssh_address expected_return_code=None, so exception not raised on nonzero return code """
        return self._ssh_address(address,
                                 command_string,
                                 expected_return_code=None)

    # TODO: reconcile this with the one in UtilityTestCase, ideally all remote
    # operations would flow through here to avoid rogue SSH calls
    def _ssh_address(self,
                     address,
                     command,
                     expected_return_code=0,
                     timeout=TEST_TIMEOUT,
                     buffer=None,
                     as_root=True):
        """
        Executes a command on a remote server over ssh.

        Sends a command over ssh to a remote machine and returns the stdout,
        stderr, and exit status. It will verify that the exit status of the
        command matches expected_return_code unless expected_return_code=None.
        """
        def host_test(address, issue_num):
            def print_result(r):
                return "rc: %s\n\nstdout:\n%s\n\nstderr:\n%s" % \
                       (r.rc, r.stdout, r.stderr)

            ping_result1 = Shell.run(['ping', '-c', '1', '-W', '1', address])
            ping_result2_report = ""
            ip_addr_result = Shell.run(['ip', 'addr', 'ls'])
            ip_route_ls_result = Shell.run(['ip', 'route', 'ls'])

            try:
                gw = [
                    l for l in ip_route_ls_result.stdout.split('\n')
                    if l.startswith("default ")
                ][0].split()[2]
                ping_gw_result = Shell.run(['ping', '-c', '1', '-W', '1', gw])
                ping_gw_report = "\nping gateway (%s): %s" % \
                    (gw, print_result(ping_gw_result))
            except:
                ping_gw_report = "\nUnable to ping gatewy.  " \
                                 "No gateway could be found in:\n" % \
                                 ip_route_ls_result.stdout

            if ping_result1.rc != 0:
                time.sleep(30)
                ping_result2 = Shell.run(
                    ['ping', '-c', '1', '-W', '1', address])
                ping_result2_report = "\n30s later ping: %s" % \
                    print_result(ping_result2)

            msg = "Error connecting to %s: %s.\n" \
                  "Please add the following to " \
                  "https://github.com/intel-hpdd/intel-manager-for-lustre/issues/%s\n" \
                  "Performing some diagnostics...\n" \
                  "ping: %s\n" \
                  "ifconfig -a: %s\n" \
                  "ip route ls: %s" \
                  "%s" \
                  "%s" % \
                  (address, e,
                   issue_num,
                   print_result(ping_result1),
                   print_result(ip_addr_result),
                   print_result(ip_route_ls_result),
                   ping_gw_report,
                   ping_result2_report)

            logger.error(msg)

            DEVNULL = open(os.devnull, 'wb')
            p = subprocess.Popen(['sendmail', '-t'],
                                 stdin=subprocess.PIPE,
                                 stdout=DEVNULL,
                                 stderr=DEVNULL)
            p.communicate(input=b'To: [email protected]\n'
                          'Subject: GH#%s\n\n' % issue_num + msg)
            p.wait()
            DEVNULL.close()

        logger.debug("remote_command[%s]: %s" % (address, command))
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())

        # the set -e just sets up a fail-safe execution environment where
        # any shell commands in command that fail and are not error checked
        # cause the shell to fail, alerting the caller that one of their
        # commands failed unexpectedly
        command = "set -e; %s" % command

        # exec 0<&- being prefixed to the shell command string below closes
        # the shell's stdin as we don't expect any uses of remote_command()
        # to read from stdin
        if not buffer:
            command = "exec 0<&-; %s" % command

        args = {'username': '******'}
        # If given an ssh_config file, require that it defines
        # a private key and username for accessing this host
        config_path = config.get('ssh_config', None)
        if config_path:
            ssh_config = paramiko.SSHConfig()
            ssh_config.parse(open(config_path))

            host_config = ssh_config.lookup(address)
            address = host_config['hostname']

            if 'user' in host_config:
                args['username'] = host_config['user']
                if args['username'] != 'root' and as_root:
                    command = "sudo sh -c \"{}\"".format(
                        command.replace('"', '\\"'))

            if 'identityfile' in host_config:
                args['key_filename'] = host_config['identityfile'][0]

                # Work around paramiko issue 157, failure to parse quoted values
                # (vagrant always quotes IdentityFile)
                args['key_filename'] = args['key_filename'].strip("\"")

        logger.info("SSH address = %s, args = %s" % (address, args))

        # Create ssh connection
        try:
            ssh.connect(address, **args)
        except paramiko.ssh_exception.SSHException, e:
            host_test(address, "29")
            return Shell.RunResult(1, "", "", timeout=False)

        transport = ssh.get_transport()
        transport.set_keepalive(20)
        channel = transport.open_session()
        channel.settimeout(timeout)

        # Actually execute the command
        try:
            channel.exec_command(command)
        except paramiko.transport.Socket, e:
            host_test(address, "72")
            return Shell.RunResult(1, "", "", timeout=False)
Esempio n. 16
0
 def run(arg_list):
     values = {
         ("rpm", "-q", "kernel"): {
             "rc":
             0,
             "stdout":
             "kernel-2.6.32-358.2.1.el6.x86_64\n"
             "kernel-2.6.32-358.18.1.el6_lustre.x86_64\n",
         },
         ("rpm", "-q", "--whatprovides", "kmod-lustre"): {
             "rc": 0,
             "stdout": "kmod-lustre-1.2.3-1.el6.x86_64\n",
         },
         ("rpm", "-ql", "--whatprovides", "lustre-osd", "kmod-lustre"):
         {
             "rc":
             0,
             "stdout":
             "/lib/modules/2.6.32-358.18.1.el7_lustre.x86_64/extra/lustre/fs/lustre.ko\n"
             "/lib/modules/2.6.32-358.18.1.el7_lustre.x86_64/extra/lustre-osd-ldiskfs/fs/osd_ldiskfs.ko\n",
         },
         (
             "modinfo",
             "-n",
             "-k",
             "2.6.32-358.2.1.el6.x86_64",
             "lustre",
             "osd_ldiskfs",
         ): {
             "rc": 1,
             "stdout": ""
         },
         (
             "modinfo",
             "-n",
             "-k",
             "2.6.32-358.18.1.el6_lustre.x86_64",
             "lustre",
             "osd_ldiskfs",
         ): {
             "rc":
             0,
             "stdout":
             "/lib/modules/2.6.32-358.18.1.el7_lustre.x86_64/extra/lustre/fs/lustre.ko\n"
             "/lib/modules/2.6.32-358.18.1.el7_lustre.x86_64/extra/lustre-osd-ldiskfs/fs/osd_ldiskfs.ko\n",
         },
         ("uname", "-m"): {
             "rc": 0,
             "stdout": "x86_64\n"
         },
         ("uname", "-r"): {
             "rc": 0,
             "stdout": "2.6.32-358.2.1.el6.x86_64\n"
         },
     }
     return Shell.RunResult(
         values[tuple(arg_list)]["rc"],
         values[tuple(arg_list)]["stdout"],
         "",
         False,
     )
    def _ssh_address(self,
                     address,
                     command,
                     expected_return_code=0,
                     timeout=TEST_TIMEOUT,
                     buffer=None,
                     as_root=True):
        """
        Executes a command on a remote server over ssh.

        Sends a command over ssh to a remote machine and returns the stdout,
        stderr, and exit status. It will verify that the exit status of the
        command matches expected_return_code unless expected_return_code=None.
        """
        def host_test(address, issue_num):
            def print_result(r):
                return "rc: %s\n\nstdout:\n%s\n\nstderr:\n%s" % \
                       (r.rc, r.stdout, r.stderr)

            ping_result1 = Shell.run(['ping', '-c', '1', '-W', '1', address])
            ping_result2_report = ""
            ip_addr_result = Shell.run(['ip', 'addr', 'ls'])
            ip_route_ls_result = Shell.run(['ip', 'route', 'ls'])

            try:
                gw = [
                    l for l in ip_route_ls_result.stdout.split('\n')
                    if l.startswith("default ")
                ][0].split()[2]
                ping_gw_result = Shell.run(['ping', '-c', '1', '-W', '1', gw])
                ping_gw_report = "\nping gateway (%s): %s" % \
                    (gw, print_result(ping_gw_result))
            except:
                ping_gw_report = "\nUnable to ping gatewy.  " \
                                 "No gateway could be found in:\n" % \
                                 ip_route_ls_result.stdout

            if ping_result1.rc != 0:
                time.sleep(30)
                ping_result2 = Shell.run(
                    ['ping', '-c', '1', '-W', '1', address])
                ping_result2_report = "\n30s later ping: %s" % \
                    print_result(ping_result2)

            msg = "Error connecting to %s: %s.\n" \
                  "Please add the following to " \
                  "https://github.com/intel-hpdd/intel-manager-for-lustre/issues/%s\n" \
                  "Performing some diagnostics...\n" \
                  "ping: %s\n" \
                  "ifconfig -a: %s\n" \
                  "ip route ls: %s" \
                  "%s" \
                  "%s" % \
                  (address, e,
                   issue_num,
                   print_result(ping_result1),
                   print_result(ip_addr_result),
                   print_result(ip_route_ls_result),
                   ping_gw_report,
                   ping_result2_report)

            logger.error(msg)

            DEVNULL = open(os.devnull, 'wb')
            p = subprocess.Popen(['sendmail', '-t'],
                                 stdin=subprocess.PIPE,
                                 stdout=DEVNULL,
                                 stderr=DEVNULL)
            p.communicate(input=b'To: [email protected]\n'
                          'Subject: GH#%s\n\n' % issue_num + msg)
            p.wait()
            DEVNULL.close()

        logger.debug("remote_command[%s]: %s" % (address, command))
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())

        # the set -e just sets up a fail-safe execution environment where
        # any shell commands in command that fail and are not error checked
        # cause the shell to fail, alerting the caller that one of their
        # commands failed unexpectedly
        command = "set -e; %s" % command

        # exec 0<&- being prefixed to the shell command string below closes
        # the shell's stdin as we don't expect any uses of remote_command()
        # to read from stdin
        if not buffer:
            command = "exec 0<&-; %s" % command

        args = {'username': '******'}
        # If given an ssh_config file, require that it defines
        # a private key and username for accessing this host
        config_path = config.get('ssh_config', None)
        if config_path:
            ssh_config = paramiko.SSHConfig()
            ssh_config.parse(open(config_path))

            host_config = ssh_config.lookup(address)
            address = host_config['hostname']

            if 'user' in host_config:
                args['username'] = host_config['user']
                if args['username'] != 'root' and as_root:
                    command = "sudo sh -c \"{}\"".format(
                        command.replace('"', '\\"'))

            if 'identityfile' in host_config:
                args['key_filename'] = host_config['identityfile'][0]

                # Work around paramiko issue 157, failure to parse quoted values
                # (vagrant always quotes IdentityFile)
                args['key_filename'] = args['key_filename'].strip("\"")

        logger.info("SSH address = %s, args = %s" % (address, args))

        # Create ssh connection
        try:
            ssh.connect(address, **args)
        except paramiko.ssh_exception.SSHException, e:
            host_test(address, "29")
            return Shell.RunResult(1, "", "", timeout=False)
Esempio n. 18
0
    def _ssh_address(self,
                     address,
                     command,
                     expected_return_code=0,
                     timeout=TEST_TIMEOUT,
                     buffer=None,
                     as_root=True):
        """
        Executes a command on a remote server over ssh.

        Sends a command over ssh to a remote machine and returns the stdout,
        stderr, and exit status. It will verify that the exit status of the
        command matches expected_return_code unless expected_return_code=None.
        """
        def host_test(address, issue_num):
            def print_result(r):
                return "rc: %s\n\nstdout:\n%s\n\nstderr:\n%s" % (
                    r.rc, r.stdout, r.stderr)

            ping_result1 = Shell.run(["ping", "-c", "1", "-W", "1", address])
            ping_result2_report = ""
            ip_addr_result = Shell.run(["ip", "addr", "ls"])
            ip_route_ls_result = Shell.run(["ip", "route", "ls"])

            try:
                gw = [
                    l for l in ip_route_ls_result.stdout.split("\n")
                    if l.startswith("default ")
                ][0].split()[2]
                ping_gw_result = Shell.run(["ping", "-c", "1", "-W", "1", gw])
                ping_gw_report = "\nping gateway (%s): %s" % (
                    gw, print_result(ping_gw_result))
            except:
                ping_gw_report = ("\nUnable to ping gatewy.  "
                                  "No gateway could be found in:\n" %
                                  ip_route_ls_result.stdout)

            if ping_result1.rc != 0:
                time.sleep(30)
                ping_result2 = Shell.run(
                    ["ping", "-c", "1", "-W", "1", address])
                ping_result2_report = "\n30s later ping: %s" % print_result(
                    ping_result2)

            msg = (
                "Error connecting to %s: %s.\n"
                "Please add the following to "
                "https://github.com/whamcloud/integrated-manager-for-lustre/issues/%s\n"
                "Performing some diagnostics...\n"
                "ping: %s\n"
                "ifconfig -a: %s\n"
                "ip route ls: %s"
                "%s"
                "%s" % (
                    address,
                    e,
                    issue_num,
                    print_result(ping_result1),
                    print_result(ip_addr_result),
                    print_result(ip_route_ls_result),
                    ping_gw_report,
                    ping_result2_report,
                ))

            logger.error(msg)

            DEVNULL = open(os.devnull, "wb")
            p = subprocess.Popen(["sendmail", "-t"],
                                 stdin=subprocess.PIPE,
                                 stdout=DEVNULL,
                                 stderr=DEVNULL)
            p.communicate(input=b"To: [email protected]\n"
                          b"Subject: GH#%s\n\n" % issue_num + msg)
            p.wait()
            DEVNULL.close()

        logger.debug("remote_command[%s]: %s" % (address, command))
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())

        # the set -e just sets up a fail-safe execution environment where
        # any shell commands in command that fail and are not error checked
        # cause the shell to fail, alerting the caller that one of their
        # commands failed unexpectedly
        command = "set -e; %s" % command

        # exec 0<&- being prefixed to the shell command string below closes
        # the shell's stdin as we don't expect any uses of remote_command()
        # to read from stdin
        if not buffer:
            command = "exec 0<&-; %s" % command

        args = {"username": "******"}
        # If given an ssh_config file, require that it defines
        # a private key and username for accessing this host
        config_path = config.get("ssh_config", None)
        if config_path:
            ssh_config = paramiko.SSHConfig()
            ssh_config.parse(open(config_path))

            host_config = ssh_config.lookup(address)
            address = host_config["hostname"]

            if "user" in host_config:
                args["username"] = host_config["user"]
                if args["username"] != "root" and as_root:
                    command = 'sudo sh -c "{}"'.format(
                        command.replace('"', '\\"'))

            if "identityfile" in host_config:
                args["key_filename"] = host_config["identityfile"][0]

                # Work around paramiko issue 157, failure to parse quoted values
                # (vagrant always quotes IdentityFile)
                args["key_filename"] = args["key_filename"].strip('"')

        logger.info(
            "SSH address = %s, timeout = %d, write len = %d, args = %s" %
            (address, timeout, len(buffer or ""), args))

        # Create ssh connection
        try:
            ssh.connect(address, **args)
        except paramiko.ssh_exception.SSHException as e:
            host_test(address, "29")
            return Shell.RunResult(1, "", "", timeout=False)

        transport = ssh.get_transport()
        transport.set_keepalive(20)
        channel = transport.open_session()
        channel.settimeout(timeout)

        # Actually execute the command
        try:
            channel.exec_command(command)
        except paramiko.transport.Socket as e:
            host_test(address, "72")
            return Shell.RunResult(1, "", "", timeout=False)

        if buffer:
            stdin = channel.makefile("wb")
            stdin.write(buffer)
            stdin.close()
        # Always shutdown write to ensure executable does not wait on input
        channel.shutdown_write()

        # Store results. This needs to happen in this order. If recv_exit_status is
        # read first, it can block indefinitely due to paramiko bug #448. The read on
        # the stdout will wait until the command finishes, so its not necessary to have
        # recv_exit_status to block on it first. Closing the channel must come last,
        # or else reading from stdout/stderr will return an empty string.
        stdout = channel.makefile("rb").read()
        stderr = channel.makefile_stderr("rb").read()
        rc = channel.recv_exit_status()
        channel.close()

        # Verify we recieved the correct exit status if one was specified.
        if expected_return_code is not None:
            self._test_case.assertEqual(
                rc,
                expected_return_code,
                "rc (%s) != expected_return_code (%s), stdout: '%s', stderr: '%s'"
                % (rc, expected_return_code, stdout, stderr),
            )

        return Shell.RunResult(rc, stdout, stderr, timeout=False)
 def run(arg_list):
     values = {("getenforce", ): ""}
     return Shell.RunResult(127, values[tuple(arg_list)],
                            "getenforce: command not found", False)