def run(self, cmd, background=False, cmdprefix=None):
        '''
        Run a ipmitool cmd.

        :throws: :class:`common.Execptions.CommandFailed`
        '''
        if cmdprefix:
            cmd = cmdprefix + self.binary + self.arguments() + cmd
        else:
            cmd = self.binary + self.arguments() + cmd
        log.debug(cmd)
        if background:
            try:
                child = subprocess.Popen(cmd, shell=True)
            except Exception as e:
                raise CommandFailed("Unable to spawn process {}".format(cmd),
                                    e, -1)
            return child
        else:
            # TODO - need python 2.7
            # output = check_output(cmd, stderr=subprocess.STDOUT, shell=True)
            try:
                cmd = subprocess.Popen(cmd,
                                       stderr=subprocess.STDOUT,
                                       stdout=subprocess.PIPE,
                                       shell=True)
            except:
                raise CommandFailed(cmd, "Failed to spawn subprocess", -1)
            output = cmd.communicate()[0]
            return output
Example #2
0
    def connect(self):
        if self.state == IPMIConsoleState.CONNECTED:
            rc_child = self.close()
        else:
            self.util.clear_state(self)

        try:
            self.ipmitool.run('sol deactivate')
        except OpTestError:
            log.info('SOL already deactivated')

        cmd = self.ipmitool.binary_name() + self.ipmitool.arguments(
        ) + ' sol activate'
        try:
            self.pty = OPexpect.spawn(
                cmd,
                failure_callback=set_system_to_UNKNOWN_BAD,
                failure_callback_data=self.system)
        except Exception as e:
            self.state = IPMIConsoleState.DISCONNECTED
            raise CommandFailed(
                'OPexpect.spawn',
                "OPexpect.spawn encountered a problem, command was '{}'".
                format(cmd), -1)

        log.debug("#IPMI SOL CONNECT")
        self.state = IPMIConsoleState.CONNECTED
        self.pty.setwinsize(1000, 1000)
        self.pty.logfile_read = OpTestLogger.FileLikeLogger(log)
        if self.delaybeforesend:
            self.pty.delaybeforesend = self.delaybeforesend
        rc = self.pty.expect_exact([
            '[SOL Session operational.  Use ~? for help]', pexpect.TIMEOUT,
            pexpect.EOF
        ],
                                   timeout=10)
        if rc == 0:
            if self.system.SUDO_set != 1 or self.system.LOGIN_set != 1 or self.system.PS1_set != 1:
                self.util.setup_term(self.system, self.pty, None,
                                     self.system.block_setup_term)
            time.sleep(0.2)
            return self.pty
        if rc == 1:
            self.pty.close()
            time.sleep(60)  # give things a minute to clear
            raise CommandFailed(
                'sol activate',
                "IPMI: pexpect.TIMEOUT while trying to establish"
                " connection, command was '{}'".format(cmd), -1)
        if rc == 2:
            self.pty.close()
            time.sleep(60)  # give things a minute to clear
            raise CommandFailed(
                'sol activate',
                "IPMI: insufficient resources for session, unable"
                " to establish IPMI v2 / RMCP+ session, command was '{}'".
                format(cmd), -1)
Example #3
0
 def try_command(self, term_obj, command, timeout=60):
     running_sudo_s = False
     extra_sudo_output = False
     expect_prompt = self.build_prompt(term_obj.prompt) + "$"
     my_term = term_obj.get_console() # if previous caller environment leaves buffer hung can show up here, e.g. PS2 prompt
     my_term.sendline(command)
     if command == 'sudo -s':
       running_sudo_s = True
       # special case to catch loss of env
       rc = my_term.expect([".*#", r"[Pp]assword for", pexpect.TIMEOUT, pexpect.EOF], timeout=timeout)
     else:
       rc = my_term.expect([expect_prompt, r"[Pp]assword for", pexpect.TIMEOUT, pexpect.EOF], timeout=timeout)
     output_list = []
     output_list += my_term.before.splitlines()
     try:
       del output_list[:1] # remove command from the list
     except Exception as e:
       pass # nothing there
     # if we are running 'sudo -s' as root then catch on generic # prompt, restore env
     if running_sudo_s and (rc == 0):
       extra_sudo_output = True
       set_env_list = self.set_env(term_obj, my_term)
     if rc == 0:
       if extra_sudo_output:
         output_list += set_env_list
       my_term.sendline("echo $?")
       rc2 = my_term.expect([expect_prompt, pexpect.TIMEOUT, pexpect.EOF], timeout=timeout)
       if rc2 == 0:
         echo_output = []
         echo_output += my_term.before.splitlines()
         try:
             del echo_output[:1] # remove command from the list
         except Exception as e:
             pass # nothing there
         try:
             echo_rc = int(echo_output[-1])
         except Exception as e:
             echo_rc = -1
       else:
         raise CommandFailed(command, "run_command echo TIMEOUT, the command may have been ok,"
                 " but unable to get echo output to confirm result", -1)
     elif rc == 1:
       handle_output_list, echo_rc = self.handle_password(term_obj, my_term, command)
       # remove the expect prompt since matched generic #
       del handle_output_list[-1]
       output_list = handle_output_list
     elif rc == 2: # timeout
         output_list, echo_rc = self.try_sendcontrol(term_obj, command) # original raw buffer if it holds any clues
     else:
       term_obj.close()
       raise CommandFailed(command, "run_command TIMEOUT or EOF, the command timed out or something,"
               " probably a connection issue, retry", -1)
     res = output_list
     if echo_rc != 0:
       raise CommandFailed(command, res, echo_rc)
     return res
Example #4
0
 def try_sendcontrol(self, term_obj, command, counter=3):
     my_term = term_obj.get_console()
     res = my_term.before
     log.warning("OpTestSystem detected something, working on recovery")
     my_term.sendcontrol('c')
     time.sleep(1)
     try_list = []
     rc = my_term.expect([".*#", pexpect.TIMEOUT, pexpect.EOF], timeout=10)
     if rc != 0:
         term_obj.close()
         self.try_recover(term_obj, counter)
         # if we get back here we still fail but have a working prompt to give back
         log.warning(
             "OpTestSystem recovered from temporary issue, but the command output is unavailable,"
             " raised Exception CommandFailed but continuing")
         raise CommandFailed(
             command,
             "run_command TIMEOUT in try_sendcontrol, we recovered the prompt,"
             " but the command output is unavailable", -1)
     else:
         # may have lost prompt
         log.warning(
             'OpTestSystem recovered from a temporary issue, continuing')
         try_list = res.splitlines()  # give back what we do have for triage
         echo_rc = 1
     return try_list, echo_rc
Example #5
0
    def retry_password(self, term_obj, my_term, command):
        retry_list_output = []
        a = 0
        while a < 3:
          a += 1
          my_term.sendline(term_obj.system.cv_HOST.password())
          rc = my_term.expect([".*#", "try again.", pexpect.TIMEOUT, pexpect.EOF])
          if (rc == 0) or (rc == 1):
            combo_io = my_term.before + my_term.after
            retry_list_output += combo_io.splitlines()
            matching = [xs for xs in sudo_responses if any(xs in xa for xa in my_term.after.splitlines())]
            if len(matching):
              echo_rc = 1
              rc = -1 # use to flag the failure next
          if rc == 0:
            retry_list_output += self.set_env(term_obj, my_term)
            echo_rc = 0
            break
          elif a == 2:
            echo_rc = 1
            break
          elif (rc == 2):
            raise CommandFailed(command, 'Retry Password TIMEOUT ' + ''.join(retry_list_output), -1)
          elif (rc == 3):
            term_obj.close()
            raise ConsoleSettings(before=my_term.before, after=my_term.after,
                    msg='SSH session/console issue, probably connection issue, retry')

        return retry_list_output, echo_rc
Example #6
0
    def run_command(self, command, timeout=60):
        console = self.get_console()
        BMC_DISCONNECT = 'SOL session closed by BMC'
        try:
            console.sendline(command)
            console.expect("\n")  # from us
            rc = console.expect([BMC_DISCONNECT, "\[console-pexpect\]#$"],
                                timeout)
            if rc == 0:
                raise BMCDisconnected(BMC_DISCONNECT)
            output = console.before
            console.sendline("echo $?")
            rc = console.expect([BMC_DISCONNECT, "\n"])  # from us
            if rc == 0:
                raise BMCDisconnected(BMC_DISCONNECT)
            rc = console.expect([BMC_DISCONNECT, "\[console-pexpect\]#$"],
                                timeout)
            if rc == 0:
                raise BMCDisconnected(BMC_DISCONNECT)
            exitcode = int(console.before)
            print "# LAST COMMAND EXIT CODE %d (%s)" % (exitcode,
                                                        repr(console.before))
        except pexpect.TIMEOUT as e:
            print e
            print console
            self.terminate()
            raise e
        except BMCDisconnected as e:
            print "# %s" % str(e)
            print "# We can possibly continue..."
            print "# Failing current command and attempting to continue"
            self.terminate()
            raise CommandFailed("ipmitool", BMC_DISCONNECT, -1)

        if rc == 0:
            res = output
            res = res.splitlines()
            if exitcode != 0:
                raise CommandFailed(command, res, exitcode)
            return res
        else:
            res = console.before
            res = res.split(command)
            return res[-1].splitlines()
Example #7
0
 def run_command(self, command, timeout=300):
     c = self.get_console()
     c.sendline(command)
     c.expect("\n")  # from us
     c.expect(c.PROMPT, timeout=timeout)
     output = c.before.splitlines()
     c.sendline("echo $?")
     c.prompt(timeout)
     exitcode = int(''.join(c.before.splitlines()[1:]))
     if exitcode != 0:
         raise CommandFailed(command, output, exitcode)
     return output
    def run_command(self, command, timeout=60):
        console = self.get_console()
        console.sendline(command)
        console.expect("\n")  # from us
        rc = None
        output = None
        exitcode = None
        try:
            rc = console.expect(["\[console-pexpect\]#$"], timeout)
            output = console.before
            console.sendline("echo $?")
            console.expect("\n")  # from us
            rc = console.expect(["\[console-pexpect\]#$"], timeout)
            exitcode = int(console.before)
        except pexpect.TIMEOUT as e:
            print e
            print "# TIMEOUT waiting for command to finish."
            print "# Attempting to control-c"
            try:
                console.sendcontrol('c')
                rc = console.expect(["\[console-pexpect\]#$"], 10)
                if rc == 0:
                    raise CommandFailed(command, "TIMEOUT", -1)
            except pexpect.TIMEOUT:
                print "# Timeout trying to kill timed-out command."
                print "# Failing current command and attempting to continue"
                self.terminate()
                raise CommandFailed("ssh -p 2222", "timeout", -1)
            raise e

        if rc == 0:
            res = output
            res = res.splitlines()
            if exitcode != 0:
                raise CommandFailed(command, res, exitcode)
            return res
        else:
            res = console.before
            res = res.split(command)
            return res[-1].splitlines()
Example #9
0
    def connect(self, logger=None):
        if self.state == ConsoleState.CONNECTED:
            rc_child = self.close()
            self.state = ConsoleState.DISCONNECTED
        else:
            self.util.clear_state(self)  # clear when coming in DISCONNECTED

        cmd = ("sshpass -p %s " % (self.password) + " ssh" +
               " -p %s" % str(self.port) + " -l %s %s" %
               (self.username, self.host) +
               " -o PubkeyAuthentication=no -o afstokenpassing=no")

        if not self.check_ssh_keys:
            cmd = (cmd + " -q" + " -o 'UserKnownHostsFile=/dev/null' " +
                   " -o 'StrictHostKeyChecking=no'")
        elif self.known_hosts_file:
            cmd = (cmd + " -o UserKnownHostsFile=" + self.known_hosts_file)

        # For multi threades SSH sessions use individual logger and file handlers per session.
        if logger:
            self.log = logger
        elif self.use_parent_logger:
            self.log = log
        else:
            self.log = OpTestLogger.optest_logger_glob.get_custom_logger(
                __name__)

        self.log.debug(cmd)

        try:
            self.pty = OPexpect.spawn(
                cmd,
                logfile=self.logfile,
                failure_callback=set_system_to_UNKNOWN_BAD,
                failure_callback_data=self.system)
        except Exception as e:
            self.state = ConsoleState.DISCONNECTED
            raise CommandFailed("OPexepct.spawn encountered a problem", e, -1)

        self.state = ConsoleState.CONNECTED
        # set for bash, otherwise it takes the 24x80 default
        self.pty.setwinsize(1000, 1000)
        if self.delaybeforesend:
            self.pty.delaybeforesend = self.delaybeforesend
        self.pty.logfile_read = OpTestLogger.FileLikeLogger(self.log)
        time.sleep(
            2
        )  # delay here in case messages like afstokenpassing unsupported show up which mess up setup_term
        self.check_set_term()
        log.debug("CONNECT starts Expect Buffer ID={}".format(hex(id(
            self.pty))))
        return self.pty
Example #10
0
    def connect(self):
        if self.state == ConsoleState.CONNECTED:
            rc_child = self.close()
            self.state = ConsoleState.DISCONNECTED
        else:
            self.util.clear_state(self)  # clear when coming in DISCONNECTED

        cmd = ("sshpass -p %s " % (self.password) + " ssh" +
               " -p %s" % str(self.port) + " -l %s %s" %
               (self.username, self.host) +
               " -o PubkeyAuthentication=no -o afstokenpassing=no")

        if not self.check_ssh_keys:
            cmd = (cmd + " -q" + " -o 'UserKnownHostsFile=/dev/null' " +
                   " -o 'StrictHostKeyChecking=no'")
        elif self.known_hosts_file:
            cmd = (cmd + " -o UserKnownHostsFile=" + self.known_hosts_file)

        log.debug(cmd)

        try:
            consoleChild = OPexpect.spawn(
                cmd,
                logfile=self.logfile,
                failure_callback=set_system_to_UNKNOWN_BAD,
                failure_callback_data=self.system)
        except Exception as e:
            self.state = ConsoleState.DISCONNECTED
            raise CommandFailed('OPexepct.spawn encountered a problem', -1)

        self.state = ConsoleState.CONNECTED
        # set for bash, otherwise it takes the 24x80 default
        consoleChild.setwinsize(1000, 1000)
        self.console = consoleChild
        if self.delaybeforesend:
            self.console.delaybeforesend = self.delaybeforesend
        # Users expecting "Host IPMI" will reference console.sol so make it available
        self.sol = self.console
        consoleChild.logfile_read = OpTestLogger.FileLikeLogger(log)
        time.sleep(
            2
        )  # delay here in case messages like afstokenpassing unsupported show up which mess up setup_term
        self.check_set_term()
        return consoleChild
Example #11
0
 def handle_password(self, term_obj, my_term, command):
     # this is for run_command 'sudo -s' or the like
     handle_list_output = []
     failure_list_output = []
     pre_combo_io = my_term.before + my_term.after
     my_term.sendline(term_obj.system.cv_HOST.password())
     rc = my_term.expect(
         [".*#$", "try again.", pexpect.TIMEOUT, pexpect.EOF])
     if (rc == 0) or (rc == 1):
         combo_io = pre_combo_io + my_term.before + my_term.after
         handle_list_output += combo_io.replace("\r\r\n", "\n").splitlines()
         matching = [
             xs for xs in sudo_responses
             if any(xs in xa for xa in my_term.after.splitlines())
         ]
         if len(matching):
             # remove the expect prompt since matched generic #
             del handle_list_output[-1]
             echo_rc = 1
             rc = -1  # use this to flag the failure next
     if rc == 0:
         # with unknown prompts and unknown environment unable to capture echo $?
         echo_rc = 0
         self.set_env(term_obj, my_term)
         list_output = handle_list_output
     elif rc == 1:
         retry_list_output, echo_rc = self.retry_password(
             term_obj, my_term, command)
         list_output = (handle_list_output + retry_list_output)
     else:
         if (rc == 2) or (rc == 3):
             failure_list_output += ['Password Problem/TIMEOUT ']
             failure_list_output += pre_combo_io.replace("\r\r\n",
                                                         "\n").splitlines()
         # timeout path needs access to output
         # handle_list_output empty if timeout or EOF
         failure_list_output += handle_list_output
         if (rc == 3):
             term_obj.close()
             raise SSHSessionDisconnected(
                 "SSH session/console exited early!")
         else:
             raise CommandFailed(command, ''.join(failure_list_output), -1)
     return list_output, echo_rc
Example #12
0
    def run_command(self, command, timeout=60):
        console = self.get_console()
        console.sendline(command)
        console.expect("\n")  # from us
        rc = console.expect(["\[console-pexpect\]#$", pexpect.TIMEOUT],
                            timeout)
        output = console.before

        console.sendline("echo $?")
        console.expect("\n")  # from us
        rc = console.expect(["\[console-pexpect\]#$", pexpect.TIMEOUT],
                            timeout)
        exitcode = int(console.before)

        if rc == 0:
            res = output
            res = res.splitlines()
            if exitcode != 0:
                raise CommandFailed(command, res, exitcode)
            return res
        else:
            res = console.before
            res = res.split(command)
            return res[-1].splitlines()
Example #13
0
    def run_command(self, command, timeout=60):
        console = self.get_console()
        BMC_DISCONNECT = 'SOL session closed by BMC'
        try:
            console.sendline(command)
            console.expect("\n") # from us
            rc = console.expect([BMC_DISCONNECT, "\[console-pexpect\]#$"], timeout)
            if rc == 0:
                raise BMCDisconnected(BMC_DISCONNECT)
            output = console.before
            console.sendline("echo $?")
            rc = console.expect([BMC_DISCONNECT, "\n"]) # from us
            if rc == 0:
                raise BMCDisconnected(BMC_DISCONNECT)
            rc = console.expect([BMC_DISCONNECT, "\[console-pexpect\]#$"], timeout)
            if rc == 0:
                raise BMCDisconnected(BMC_DISCONNECT)
            exitcode = int(console.before)
            print "# LAST COMMAND EXIT CODE %d (%s)" % (exitcode, repr(console.before))
        except pexpect.TIMEOUT as e:
            print e
            print "# TIMEOUT waiting for command to finish."
            print "# Attempting to control-c"
            try:
                console.sendcontrol('c')
                rc = console.expect([BMC_DISCONNECT, "\[console-pexpect\]#$"], 10)
                if rc == 0:
                    print "# BMC Disconnect while cancelling timed-out command"
                    print "# Failing test, disconnecting/reconnecting"
                    self.terminate()
                    raise CommandFailed("ipmitool", BMC_DISCONNECT, -1)
                if rc == 1:
                    raise CommandFailed(command, "TIMEOUT", -1)
            except pexpect.TIMEOUT:
                print "# Timeout trying to kill timed-out command."
                print "# Failing current command and attempting to continue"
                self.terminate()
                raise CommandFailed("ipmitool", "timeout", -1)
            raise e
        except BMCDisconnected as e:
            print "# %s" % str(e)
            print "# We can possibly continue..."
            print "# Failing current command and attempting to continue"
            self.terminate()
            self.connect()
            console = self.get_console()
            print "# On reconnect, attempt to cancel last command (ctrl-c)"
            # Note: this is a terrible idea. If BMC vendors created reliable
            # SoL implementations this kind of crap wouldn't be needed.
            # This is incorrect on so many levels it's not funny. For a start,
            # we really don't want to send random control characters during
            # boot!
            console.sendcontrol('c')
            try:
                rc = console.expect([BMC_DISCONNECT,
                                     "\[console-pexpect\]#$"], 10)
                if rc == 0:
                    self.terminate()
                    raise BMCDisconnected(BMC_DISCONNECT)
            except pexpect.TIMEOUT:
                print "# No response from BMC... trying 'mc reset cold'"
                self.mc_reset()
                self.terminate()
                self.connect()
                console = self.get_console()
                console.sendcontrol('c')
            raise CommandFailed("ipmitool", BMC_DISCONNECT, -1)

        if rc == 1:
            res = output
            res = res.splitlines()
            if exitcode != 0:
                raise CommandFailed(command, res, exitcode)
            return res
        else:
            res = console.before
            res = res.split(command)
            return res[-1].splitlines()