def run(self, cmd, background=False, cmdprefix=None): ''' Run a ipmitool cmd. :throws: :class:`common.Execptions.CommandFailed` ''' if cmdprefix: cmd = cmdprefix + self.binary + self.arguments() + cmd else: cmd = self.binary + self.arguments() + cmd log.debug(cmd) if background: try: child = subprocess.Popen(cmd, shell=True) except Exception as e: raise CommandFailed("Unable to spawn process {}".format(cmd), e, -1) return child else: # TODO - need python 2.7 # output = check_output(cmd, stderr=subprocess.STDOUT, shell=True) try: cmd = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True) except: raise CommandFailed(cmd, "Failed to spawn subprocess", -1) output = cmd.communicate()[0] return output
def connect(self): if self.state == IPMIConsoleState.CONNECTED: rc_child = self.close() else: self.util.clear_state(self) try: self.ipmitool.run('sol deactivate') except OpTestError: log.info('SOL already deactivated') cmd = self.ipmitool.binary_name() + self.ipmitool.arguments( ) + ' sol activate' try: self.pty = OPexpect.spawn( cmd, failure_callback=set_system_to_UNKNOWN_BAD, failure_callback_data=self.system) except Exception as e: self.state = IPMIConsoleState.DISCONNECTED raise CommandFailed( 'OPexpect.spawn', "OPexpect.spawn encountered a problem, command was '{}'". format(cmd), -1) log.debug("#IPMI SOL CONNECT") self.state = IPMIConsoleState.CONNECTED self.pty.setwinsize(1000, 1000) self.pty.logfile_read = OpTestLogger.FileLikeLogger(log) if self.delaybeforesend: self.pty.delaybeforesend = self.delaybeforesend rc = self.pty.expect_exact([ '[SOL Session operational. Use ~? for help]', pexpect.TIMEOUT, pexpect.EOF ], timeout=10) if rc == 0: if self.system.SUDO_set != 1 or self.system.LOGIN_set != 1 or self.system.PS1_set != 1: self.util.setup_term(self.system, self.pty, None, self.system.block_setup_term) time.sleep(0.2) return self.pty if rc == 1: self.pty.close() time.sleep(60) # give things a minute to clear raise CommandFailed( 'sol activate', "IPMI: pexpect.TIMEOUT while trying to establish" " connection, command was '{}'".format(cmd), -1) if rc == 2: self.pty.close() time.sleep(60) # give things a minute to clear raise CommandFailed( 'sol activate', "IPMI: insufficient resources for session, unable" " to establish IPMI v2 / RMCP+ session, command was '{}'". format(cmd), -1)
def try_command(self, term_obj, command, timeout=60): running_sudo_s = False extra_sudo_output = False expect_prompt = self.build_prompt(term_obj.prompt) + "$" my_term = term_obj.get_console() # if previous caller environment leaves buffer hung can show up here, e.g. PS2 prompt my_term.sendline(command) if command == 'sudo -s': running_sudo_s = True # special case to catch loss of env rc = my_term.expect([".*#", r"[Pp]assword for", pexpect.TIMEOUT, pexpect.EOF], timeout=timeout) else: rc = my_term.expect([expect_prompt, r"[Pp]assword for", pexpect.TIMEOUT, pexpect.EOF], timeout=timeout) output_list = [] output_list += my_term.before.splitlines() try: del output_list[:1] # remove command from the list except Exception as e: pass # nothing there # if we are running 'sudo -s' as root then catch on generic # prompt, restore env if running_sudo_s and (rc == 0): extra_sudo_output = True set_env_list = self.set_env(term_obj, my_term) if rc == 0: if extra_sudo_output: output_list += set_env_list my_term.sendline("echo $?") rc2 = my_term.expect([expect_prompt, pexpect.TIMEOUT, pexpect.EOF], timeout=timeout) if rc2 == 0: echo_output = [] echo_output += my_term.before.splitlines() try: del echo_output[:1] # remove command from the list except Exception as e: pass # nothing there try: echo_rc = int(echo_output[-1]) except Exception as e: echo_rc = -1 else: raise CommandFailed(command, "run_command echo TIMEOUT, the command may have been ok," " but unable to get echo output to confirm result", -1) elif rc == 1: handle_output_list, echo_rc = self.handle_password(term_obj, my_term, command) # remove the expect prompt since matched generic # del handle_output_list[-1] output_list = handle_output_list elif rc == 2: # timeout output_list, echo_rc = self.try_sendcontrol(term_obj, command) # original raw buffer if it holds any clues else: term_obj.close() raise CommandFailed(command, "run_command TIMEOUT or EOF, the command timed out or something," " probably a connection issue, retry", -1) res = output_list if echo_rc != 0: raise CommandFailed(command, res, echo_rc) return res
def try_sendcontrol(self, term_obj, command, counter=3): my_term = term_obj.get_console() res = my_term.before log.warning("OpTestSystem detected something, working on recovery") my_term.sendcontrol('c') time.sleep(1) try_list = [] rc = my_term.expect([".*#", pexpect.TIMEOUT, pexpect.EOF], timeout=10) if rc != 0: term_obj.close() self.try_recover(term_obj, counter) # if we get back here we still fail but have a working prompt to give back log.warning( "OpTestSystem recovered from temporary issue, but the command output is unavailable," " raised Exception CommandFailed but continuing") raise CommandFailed( command, "run_command TIMEOUT in try_sendcontrol, we recovered the prompt," " but the command output is unavailable", -1) else: # may have lost prompt log.warning( 'OpTestSystem recovered from a temporary issue, continuing') try_list = res.splitlines() # give back what we do have for triage echo_rc = 1 return try_list, echo_rc
def retry_password(self, term_obj, my_term, command): retry_list_output = [] a = 0 while a < 3: a += 1 my_term.sendline(term_obj.system.cv_HOST.password()) rc = my_term.expect([".*#", "try again.", pexpect.TIMEOUT, pexpect.EOF]) if (rc == 0) or (rc == 1): combo_io = my_term.before + my_term.after retry_list_output += combo_io.splitlines() matching = [xs for xs in sudo_responses if any(xs in xa for xa in my_term.after.splitlines())] if len(matching): echo_rc = 1 rc = -1 # use to flag the failure next if rc == 0: retry_list_output += self.set_env(term_obj, my_term) echo_rc = 0 break elif a == 2: echo_rc = 1 break elif (rc == 2): raise CommandFailed(command, 'Retry Password TIMEOUT ' + ''.join(retry_list_output), -1) elif (rc == 3): term_obj.close() raise ConsoleSettings(before=my_term.before, after=my_term.after, msg='SSH session/console issue, probably connection issue, retry') return retry_list_output, echo_rc
def run_command(self, command, timeout=60): console = self.get_console() BMC_DISCONNECT = 'SOL session closed by BMC' try: console.sendline(command) console.expect("\n") # from us rc = console.expect([BMC_DISCONNECT, "\[console-pexpect\]#$"], timeout) if rc == 0: raise BMCDisconnected(BMC_DISCONNECT) output = console.before console.sendline("echo $?") rc = console.expect([BMC_DISCONNECT, "\n"]) # from us if rc == 0: raise BMCDisconnected(BMC_DISCONNECT) rc = console.expect([BMC_DISCONNECT, "\[console-pexpect\]#$"], timeout) if rc == 0: raise BMCDisconnected(BMC_DISCONNECT) exitcode = int(console.before) print "# LAST COMMAND EXIT CODE %d (%s)" % (exitcode, repr(console.before)) except pexpect.TIMEOUT as e: print e print console self.terminate() raise e except BMCDisconnected as e: print "# %s" % str(e) print "# We can possibly continue..." print "# Failing current command and attempting to continue" self.terminate() raise CommandFailed("ipmitool", BMC_DISCONNECT, -1) if rc == 0: res = output res = res.splitlines() if exitcode != 0: raise CommandFailed(command, res, exitcode) return res else: res = console.before res = res.split(command) return res[-1].splitlines()
def run_command(self, command, timeout=300): c = self.get_console() c.sendline(command) c.expect("\n") # from us c.expect(c.PROMPT, timeout=timeout) output = c.before.splitlines() c.sendline("echo $?") c.prompt(timeout) exitcode = int(''.join(c.before.splitlines()[1:])) if exitcode != 0: raise CommandFailed(command, output, exitcode) return output
def run_command(self, command, timeout=60): console = self.get_console() console.sendline(command) console.expect("\n") # from us rc = None output = None exitcode = None try: rc = console.expect(["\[console-pexpect\]#$"], timeout) output = console.before console.sendline("echo $?") console.expect("\n") # from us rc = console.expect(["\[console-pexpect\]#$"], timeout) exitcode = int(console.before) except pexpect.TIMEOUT as e: print e print "# TIMEOUT waiting for command to finish." print "# Attempting to control-c" try: console.sendcontrol('c') rc = console.expect(["\[console-pexpect\]#$"], 10) if rc == 0: raise CommandFailed(command, "TIMEOUT", -1) except pexpect.TIMEOUT: print "# Timeout trying to kill timed-out command." print "# Failing current command and attempting to continue" self.terminate() raise CommandFailed("ssh -p 2222", "timeout", -1) raise e if rc == 0: res = output res = res.splitlines() if exitcode != 0: raise CommandFailed(command, res, exitcode) return res else: res = console.before res = res.split(command) return res[-1].splitlines()
def connect(self, logger=None): if self.state == ConsoleState.CONNECTED: rc_child = self.close() self.state = ConsoleState.DISCONNECTED else: self.util.clear_state(self) # clear when coming in DISCONNECTED cmd = ("sshpass -p %s " % (self.password) + " ssh" + " -p %s" % str(self.port) + " -l %s %s" % (self.username, self.host) + " -o PubkeyAuthentication=no -o afstokenpassing=no") if not self.check_ssh_keys: cmd = (cmd + " -q" + " -o 'UserKnownHostsFile=/dev/null' " + " -o 'StrictHostKeyChecking=no'") elif self.known_hosts_file: cmd = (cmd + " -o UserKnownHostsFile=" + self.known_hosts_file) # For multi threades SSH sessions use individual logger and file handlers per session. if logger: self.log = logger elif self.use_parent_logger: self.log = log else: self.log = OpTestLogger.optest_logger_glob.get_custom_logger( __name__) self.log.debug(cmd) try: self.pty = OPexpect.spawn( cmd, logfile=self.logfile, failure_callback=set_system_to_UNKNOWN_BAD, failure_callback_data=self.system) except Exception as e: self.state = ConsoleState.DISCONNECTED raise CommandFailed("OPexepct.spawn encountered a problem", e, -1) self.state = ConsoleState.CONNECTED # set for bash, otherwise it takes the 24x80 default self.pty.setwinsize(1000, 1000) if self.delaybeforesend: self.pty.delaybeforesend = self.delaybeforesend self.pty.logfile_read = OpTestLogger.FileLikeLogger(self.log) time.sleep( 2 ) # delay here in case messages like afstokenpassing unsupported show up which mess up setup_term self.check_set_term() log.debug("CONNECT starts Expect Buffer ID={}".format(hex(id( self.pty)))) return self.pty
def connect(self): if self.state == ConsoleState.CONNECTED: rc_child = self.close() self.state = ConsoleState.DISCONNECTED else: self.util.clear_state(self) # clear when coming in DISCONNECTED cmd = ("sshpass -p %s " % (self.password) + " ssh" + " -p %s" % str(self.port) + " -l %s %s" % (self.username, self.host) + " -o PubkeyAuthentication=no -o afstokenpassing=no") if not self.check_ssh_keys: cmd = (cmd + " -q" + " -o 'UserKnownHostsFile=/dev/null' " + " -o 'StrictHostKeyChecking=no'") elif self.known_hosts_file: cmd = (cmd + " -o UserKnownHostsFile=" + self.known_hosts_file) log.debug(cmd) try: consoleChild = OPexpect.spawn( cmd, logfile=self.logfile, failure_callback=set_system_to_UNKNOWN_BAD, failure_callback_data=self.system) except Exception as e: self.state = ConsoleState.DISCONNECTED raise CommandFailed('OPexepct.spawn encountered a problem', -1) self.state = ConsoleState.CONNECTED # set for bash, otherwise it takes the 24x80 default consoleChild.setwinsize(1000, 1000) self.console = consoleChild if self.delaybeforesend: self.console.delaybeforesend = self.delaybeforesend # Users expecting "Host IPMI" will reference console.sol so make it available self.sol = self.console consoleChild.logfile_read = OpTestLogger.FileLikeLogger(log) time.sleep( 2 ) # delay here in case messages like afstokenpassing unsupported show up which mess up setup_term self.check_set_term() return consoleChild
def handle_password(self, term_obj, my_term, command): # this is for run_command 'sudo -s' or the like handle_list_output = [] failure_list_output = [] pre_combo_io = my_term.before + my_term.after my_term.sendline(term_obj.system.cv_HOST.password()) rc = my_term.expect( [".*#$", "try again.", pexpect.TIMEOUT, pexpect.EOF]) if (rc == 0) or (rc == 1): combo_io = pre_combo_io + my_term.before + my_term.after handle_list_output += combo_io.replace("\r\r\n", "\n").splitlines() matching = [ xs for xs in sudo_responses if any(xs in xa for xa in my_term.after.splitlines()) ] if len(matching): # remove the expect prompt since matched generic # del handle_list_output[-1] echo_rc = 1 rc = -1 # use this to flag the failure next if rc == 0: # with unknown prompts and unknown environment unable to capture echo $? echo_rc = 0 self.set_env(term_obj, my_term) list_output = handle_list_output elif rc == 1: retry_list_output, echo_rc = self.retry_password( term_obj, my_term, command) list_output = (handle_list_output + retry_list_output) else: if (rc == 2) or (rc == 3): failure_list_output += ['Password Problem/TIMEOUT '] failure_list_output += pre_combo_io.replace("\r\r\n", "\n").splitlines() # timeout path needs access to output # handle_list_output empty if timeout or EOF failure_list_output += handle_list_output if (rc == 3): term_obj.close() raise SSHSessionDisconnected( "SSH session/console exited early!") else: raise CommandFailed(command, ''.join(failure_list_output), -1) return list_output, echo_rc
def run_command(self, command, timeout=60): console = self.get_console() console.sendline(command) console.expect("\n") # from us rc = console.expect(["\[console-pexpect\]#$", pexpect.TIMEOUT], timeout) output = console.before console.sendline("echo $?") console.expect("\n") # from us rc = console.expect(["\[console-pexpect\]#$", pexpect.TIMEOUT], timeout) exitcode = int(console.before) if rc == 0: res = output res = res.splitlines() if exitcode != 0: raise CommandFailed(command, res, exitcode) return res else: res = console.before res = res.split(command) return res[-1].splitlines()
def run_command(self, command, timeout=60): console = self.get_console() BMC_DISCONNECT = 'SOL session closed by BMC' try: console.sendline(command) console.expect("\n") # from us rc = console.expect([BMC_DISCONNECT, "\[console-pexpect\]#$"], timeout) if rc == 0: raise BMCDisconnected(BMC_DISCONNECT) output = console.before console.sendline("echo $?") rc = console.expect([BMC_DISCONNECT, "\n"]) # from us if rc == 0: raise BMCDisconnected(BMC_DISCONNECT) rc = console.expect([BMC_DISCONNECT, "\[console-pexpect\]#$"], timeout) if rc == 0: raise BMCDisconnected(BMC_DISCONNECT) exitcode = int(console.before) print "# LAST COMMAND EXIT CODE %d (%s)" % (exitcode, repr(console.before)) except pexpect.TIMEOUT as e: print e print "# TIMEOUT waiting for command to finish." print "# Attempting to control-c" try: console.sendcontrol('c') rc = console.expect([BMC_DISCONNECT, "\[console-pexpect\]#$"], 10) if rc == 0: print "# BMC Disconnect while cancelling timed-out command" print "# Failing test, disconnecting/reconnecting" self.terminate() raise CommandFailed("ipmitool", BMC_DISCONNECT, -1) if rc == 1: raise CommandFailed(command, "TIMEOUT", -1) except pexpect.TIMEOUT: print "# Timeout trying to kill timed-out command." print "# Failing current command and attempting to continue" self.terminate() raise CommandFailed("ipmitool", "timeout", -1) raise e except BMCDisconnected as e: print "# %s" % str(e) print "# We can possibly continue..." print "# Failing current command and attempting to continue" self.terminate() self.connect() console = self.get_console() print "# On reconnect, attempt to cancel last command (ctrl-c)" # Note: this is a terrible idea. If BMC vendors created reliable # SoL implementations this kind of crap wouldn't be needed. # This is incorrect on so many levels it's not funny. For a start, # we really don't want to send random control characters during # boot! console.sendcontrol('c') try: rc = console.expect([BMC_DISCONNECT, "\[console-pexpect\]#$"], 10) if rc == 0: self.terminate() raise BMCDisconnected(BMC_DISCONNECT) except pexpect.TIMEOUT: print "# No response from BMC... trying 'mc reset cold'" self.mc_reset() self.terminate() self.connect() console = self.get_console() console.sendcontrol('c') raise CommandFailed("ipmitool", BMC_DISCONNECT, -1) if rc == 1: res = output res = res.splitlines() if exitcode != 0: raise CommandFailed(command, res, exitcode) return res else: res = console.before res = res.split(command) return res[-1].splitlines()