def run(self, cmd, background=False, cmdprefix=None):
'''
Run a ipmitool cmd.
:throws: :class:`common.Execptions.CommandFailed`
'''
if cmdprefix:
cmd = cmdprefix + self.binary + self.arguments() + cmd
else:
cmd = self.binary + self.arguments() + cmd
log.debug(cmd)
if background:
try:
child = subprocess.Popen(cmd, shell=True)
except Exception as e:
raise CommandFailed("Unable to spawn process {}".format(cmd),
e, -1)
return child
else:
# TODO - need python 2.7
# output = check_output(cmd, stderr=subprocess.STDOUT, shell=True)
try:
cmd = subprocess.Popen(cmd,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
shell=True)
except:
raise CommandFailed(cmd, "Failed to spawn subprocess", -1)
output = cmd.communicate()[0]
return output
def connect(self):
if self.state == IPMIConsoleState.CONNECTED:
rc_child = self.close()
else:
self.util.clear_state(self)
try:
self.ipmitool.run('sol deactivate')
except OpTestError:
log.info('SOL already deactivated')
cmd = self.ipmitool.binary_name() + self.ipmitool.arguments(
) + ' sol activate'
try:
self.pty = OPexpect.spawn(
cmd,
failure_callback=set_system_to_UNKNOWN_BAD,
failure_callback_data=self.system)
except Exception as e:
self.state = IPMIConsoleState.DISCONNECTED
raise CommandFailed(
'OPexpect.spawn',
"OPexpect.spawn encountered a problem, command was '{}'".
format(cmd), -1)
log.debug("#IPMI SOL CONNECT")
self.state = IPMIConsoleState.CONNECTED
self.pty.setwinsize(1000, 1000)
self.pty.logfile_read = OpTestLogger.FileLikeLogger(log)
if self.delaybeforesend:
self.pty.delaybeforesend = self.delaybeforesend
rc = self.pty.expect_exact([
'[SOL Session operational. Use ~? for help]', pexpect.TIMEOUT,
pexpect.EOF
],
timeout=10)
if rc == 0:
if self.system.SUDO_set != 1 or self.system.LOGIN_set != 1 or self.system.PS1_set != 1:
self.util.setup_term(self.system, self.pty, None,
self.system.block_setup_term)
time.sleep(0.2)
return self.pty
if rc == 1:
self.pty.close()
time.sleep(60) # give things a minute to clear
raise CommandFailed(
'sol activate',
"IPMI: pexpect.TIMEOUT while trying to establish"
" connection, command was '{}'".format(cmd), -1)
if rc == 2:
self.pty.close()
time.sleep(60) # give things a minute to clear
raise CommandFailed(
'sol activate',
"IPMI: insufficient resources for session, unable"
" to establish IPMI v2 / RMCP+ session, command was '{}'".
format(cmd), -1)
def try_command(self, term_obj, command, timeout=60):
running_sudo_s = False
extra_sudo_output = False
expect_prompt = self.build_prompt(term_obj.prompt) + "$"
my_term = term_obj.get_console() # if previous caller environment leaves buffer hung can show up here, e.g. PS2 prompt
my_term.sendline(command)
if command == 'sudo -s':
running_sudo_s = True
# special case to catch loss of env
rc = my_term.expect([".*#", r"[Pp]assword for", pexpect.TIMEOUT, pexpect.EOF], timeout=timeout)
else:
rc = my_term.expect([expect_prompt, r"[Pp]assword for", pexpect.TIMEOUT, pexpect.EOF], timeout=timeout)
output_list = []
output_list += my_term.before.splitlines()
try:
del output_list[:1] # remove command from the list
except Exception as e:
pass # nothing there
# if we are running 'sudo -s' as root then catch on generic # prompt, restore env
if running_sudo_s and (rc == 0):
extra_sudo_output = True
set_env_list = self.set_env(term_obj, my_term)
if rc == 0:
if extra_sudo_output:
output_list += set_env_list
my_term.sendline("echo $?")
rc2 = my_term.expect([expect_prompt, pexpect.TIMEOUT, pexpect.EOF], timeout=timeout)
if rc2 == 0:
echo_output = []
echo_output += my_term.before.splitlines()
try:
del echo_output[:1] # remove command from the list
except Exception as e:
pass # nothing there
try:
echo_rc = int(echo_output[-1])
except Exception as e:
echo_rc = -1
else:
raise CommandFailed(command, "run_command echo TIMEOUT, the command may have been ok,"
" but unable to get echo output to confirm result", -1)
elif rc == 1:
handle_output_list, echo_rc = self.handle_password(term_obj, my_term, command)
# remove the expect prompt since matched generic #
del handle_output_list[-1]
output_list = handle_output_list
elif rc == 2: # timeout
output_list, echo_rc = self.try_sendcontrol(term_obj, command) # original raw buffer if it holds any clues
else:
term_obj.close()
raise CommandFailed(command, "run_command TIMEOUT or EOF, the command timed out or something,"
" probably a connection issue, retry", -1)
res = output_list
if echo_rc != 0:
raise CommandFailed(command, res, echo_rc)
return res
def try_sendcontrol(self, term_obj, command, counter=3):
my_term = term_obj.get_console()
res = my_term.before
log.warning("OpTestSystem detected something, working on recovery")
my_term.sendcontrol('c')
time.sleep(1)
try_list = []
rc = my_term.expect([".*#", pexpect.TIMEOUT, pexpect.EOF], timeout=10)
if rc != 0:
term_obj.close()
self.try_recover(term_obj, counter)
# if we get back here we still fail but have a working prompt to give back
log.warning(
"OpTestSystem recovered from temporary issue, but the command output is unavailable,"
" raised Exception CommandFailed but continuing")
raise CommandFailed(
command,
"run_command TIMEOUT in try_sendcontrol, we recovered the prompt,"
" but the command output is unavailable", -1)
else:
# may have lost prompt
log.warning(
'OpTestSystem recovered from a temporary issue, continuing')
try_list = res.splitlines() # give back what we do have for triage
echo_rc = 1
return try_list, echo_rc
def retry_password(self, term_obj, my_term, command):
retry_list_output = []
a = 0
while a < 3:
a += 1
my_term.sendline(term_obj.system.cv_HOST.password())
rc = my_term.expect([".*#", "try again.", pexpect.TIMEOUT, pexpect.EOF])
if (rc == 0) or (rc == 1):
combo_io = my_term.before + my_term.after
retry_list_output += combo_io.splitlines()
matching = [xs for xs in sudo_responses if any(xs in xa for xa in my_term.after.splitlines())]
if len(matching):
echo_rc = 1
rc = -1 # use to flag the failure next
if rc == 0:
retry_list_output += self.set_env(term_obj, my_term)
echo_rc = 0
break
elif a == 2:
echo_rc = 1
break
elif (rc == 2):
raise CommandFailed(command, 'Retry Password TIMEOUT ' + ''.join(retry_list_output), -1)
elif (rc == 3):
term_obj.close()
raise ConsoleSettings(before=my_term.before, after=my_term.after,
msg='SSH session/console issue, probably connection issue, retry')
return retry_list_output, echo_rc
def run_command(self, command, timeout=60):
console = self.get_console()
BMC_DISCONNECT = 'SOL session closed by BMC'
try:
console.sendline(command)
console.expect("\n") # from us
rc = console.expect([BMC_DISCONNECT, "\[console-pexpect\]#$"],
timeout)
if rc == 0:
raise BMCDisconnected(BMC_DISCONNECT)
output = console.before
console.sendline("echo $?")
rc = console.expect([BMC_DISCONNECT, "\n"]) # from us
if rc == 0:
raise BMCDisconnected(BMC_DISCONNECT)
rc = console.expect([BMC_DISCONNECT, "\[console-pexpect\]#$"],
timeout)
if rc == 0:
raise BMCDisconnected(BMC_DISCONNECT)
exitcode = int(console.before)
print "# LAST COMMAND EXIT CODE %d (%s)" % (exitcode,
repr(console.before))
except pexpect.TIMEOUT as e:
print e
print console
self.terminate()
raise e
except BMCDisconnected as e:
print "# %s" % str(e)
print "# We can possibly continue..."
print "# Failing current command and attempting to continue"
self.terminate()
raise CommandFailed("ipmitool", BMC_DISCONNECT, -1)
if rc == 0:
res = output
res = res.splitlines()
if exitcode != 0:
raise CommandFailed(command, res, exitcode)
return res
else:
res = console.before
res = res.split(command)
return res[-1].splitlines()
def run_command(self, command, timeout=300):
c = self.get_console()
c.sendline(command)
c.expect("\n") # from us
c.expect(c.PROMPT, timeout=timeout)
output = c.before.splitlines()
c.sendline("echo $?")
c.prompt(timeout)
exitcode = int(''.join(c.before.splitlines()[1:]))
if exitcode != 0:
raise CommandFailed(command, output, exitcode)
return output
def run_command(self, command, timeout=60):
console = self.get_console()
console.sendline(command)
console.expect("\n") # from us
rc = None
output = None
exitcode = None
try:
rc = console.expect(["\[console-pexpect\]#$"], timeout)
output = console.before
console.sendline("echo $?")
console.expect("\n") # from us
rc = console.expect(["\[console-pexpect\]#$"], timeout)
exitcode = int(console.before)
except pexpect.TIMEOUT as e:
print e
print "# TIMEOUT waiting for command to finish."
print "# Attempting to control-c"
try:
console.sendcontrol('c')
rc = console.expect(["\[console-pexpect\]#$"], 10)
if rc == 0:
raise CommandFailed(command, "TIMEOUT", -1)
except pexpect.TIMEOUT:
print "# Timeout trying to kill timed-out command."
print "# Failing current command and attempting to continue"
self.terminate()
raise CommandFailed("ssh -p 2222", "timeout", -1)
raise e
if rc == 0:
res = output
res = res.splitlines()
if exitcode != 0:
raise CommandFailed(command, res, exitcode)
return res
else:
res = console.before
res = res.split(command)
return res[-1].splitlines()
def connect(self, logger=None):
if self.state == ConsoleState.CONNECTED:
rc_child = self.close()
self.state = ConsoleState.DISCONNECTED
else:
self.util.clear_state(self) # clear when coming in DISCONNECTED
cmd = ("sshpass -p %s " % (self.password) + " ssh" +
" -p %s" % str(self.port) + " -l %s %s" %
(self.username, self.host) +
" -o PubkeyAuthentication=no -o afstokenpassing=no")
if not self.check_ssh_keys:
cmd = (cmd + " -q" + " -o 'UserKnownHostsFile=/dev/null' " +
" -o 'StrictHostKeyChecking=no'")
elif self.known_hosts_file:
cmd = (cmd + " -o UserKnownHostsFile=" + self.known_hosts_file)
# For multi threades SSH sessions use individual logger and file handlers per session.
if logger:
self.log = logger
elif self.use_parent_logger:
self.log = log
else:
self.log = OpTestLogger.optest_logger_glob.get_custom_logger(
__name__)
self.log.debug(cmd)
try:
self.pty = OPexpect.spawn(
cmd,
logfile=self.logfile,
failure_callback=set_system_to_UNKNOWN_BAD,
failure_callback_data=self.system)
except Exception as e:
self.state = ConsoleState.DISCONNECTED
raise CommandFailed("OPexepct.spawn encountered a problem", e, -1)
self.state = ConsoleState.CONNECTED
# set for bash, otherwise it takes the 24x80 default
self.pty.setwinsize(1000, 1000)
if self.delaybeforesend:
self.pty.delaybeforesend = self.delaybeforesend
self.pty.logfile_read = OpTestLogger.FileLikeLogger(self.log)
time.sleep(
2
) # delay here in case messages like afstokenpassing unsupported show up which mess up setup_term
self.check_set_term()
log.debug("CONNECT starts Expect Buffer ID={}".format(hex(id(
self.pty))))
return self.pty
def connect(self):
if self.state == ConsoleState.CONNECTED:
rc_child = self.close()
self.state = ConsoleState.DISCONNECTED
else:
self.util.clear_state(self) # clear when coming in DISCONNECTED
cmd = ("sshpass -p %s " % (self.password) + " ssh" +
" -p %s" % str(self.port) + " -l %s %s" %
(self.username, self.host) +
" -o PubkeyAuthentication=no -o afstokenpassing=no")
if not self.check_ssh_keys:
cmd = (cmd + " -q" + " -o 'UserKnownHostsFile=/dev/null' " +
" -o 'StrictHostKeyChecking=no'")
elif self.known_hosts_file:
cmd = (cmd + " -o UserKnownHostsFile=" + self.known_hosts_file)
log.debug(cmd)
try:
consoleChild = OPexpect.spawn(
cmd,
logfile=self.logfile,
failure_callback=set_system_to_UNKNOWN_BAD,
failure_callback_data=self.system)
except Exception as e:
self.state = ConsoleState.DISCONNECTED
raise CommandFailed('OPexepct.spawn encountered a problem', -1)
self.state = ConsoleState.CONNECTED
# set for bash, otherwise it takes the 24x80 default
consoleChild.setwinsize(1000, 1000)
self.console = consoleChild
if self.delaybeforesend:
self.console.delaybeforesend = self.delaybeforesend
# Users expecting "Host IPMI" will reference console.sol so make it available
self.sol = self.console
consoleChild.logfile_read = OpTestLogger.FileLikeLogger(log)
time.sleep(
2
) # delay here in case messages like afstokenpassing unsupported show up which mess up setup_term
self.check_set_term()
return consoleChild
def handle_password(self, term_obj, my_term, command):
# this is for run_command 'sudo -s' or the like
handle_list_output = []
failure_list_output = []
pre_combo_io = my_term.before + my_term.after
my_term.sendline(term_obj.system.cv_HOST.password())
rc = my_term.expect(
[".*#$", "try again.", pexpect.TIMEOUT, pexpect.EOF])
if (rc == 0) or (rc == 1):
combo_io = pre_combo_io + my_term.before + my_term.after
handle_list_output += combo_io.replace("\r\r\n", "\n").splitlines()
matching = [
xs for xs in sudo_responses
if any(xs in xa for xa in my_term.after.splitlines())
]
if len(matching):
# remove the expect prompt since matched generic #
del handle_list_output[-1]
echo_rc = 1
rc = -1 # use this to flag the failure next
if rc == 0:
# with unknown prompts and unknown environment unable to capture echo $?
echo_rc = 0
self.set_env(term_obj, my_term)
list_output = handle_list_output
elif rc == 1:
retry_list_output, echo_rc = self.retry_password(
term_obj, my_term, command)
list_output = (handle_list_output + retry_list_output)
else:
if (rc == 2) or (rc == 3):
failure_list_output += ['Password Problem/TIMEOUT ']
failure_list_output += pre_combo_io.replace("\r\r\n",
"\n").splitlines()
# timeout path needs access to output
# handle_list_output empty if timeout or EOF
failure_list_output += handle_list_output
if (rc == 3):
term_obj.close()
raise SSHSessionDisconnected(
"SSH session/console exited early!")
else:
raise CommandFailed(command, ''.join(failure_list_output), -1)
return list_output, echo_rc
def run_command(self, command, timeout=60):
console = self.get_console()
console.sendline(command)
console.expect("\n") # from us
rc = console.expect(["\[console-pexpect\]#$", pexpect.TIMEOUT],
timeout)
output = console.before
console.sendline("echo $?")
console.expect("\n") # from us
rc = console.expect(["\[console-pexpect\]#$", pexpect.TIMEOUT],
timeout)
exitcode = int(console.before)
if rc == 0:
res = output
res = res.splitlines()
if exitcode != 0:
raise CommandFailed(command, res, exitcode)
return res
else:
res = console.before
res = res.split(command)
return res[-1].splitlines()
def run_command(self, command, timeout=60):
console = self.get_console()
BMC_DISCONNECT = 'SOL session closed by BMC'
try:
console.sendline(command)
console.expect("\n") # from us
rc = console.expect([BMC_DISCONNECT, "\[console-pexpect\]#$"], timeout)
if rc == 0:
raise BMCDisconnected(BMC_DISCONNECT)
output = console.before
console.sendline("echo $?")
rc = console.expect([BMC_DISCONNECT, "\n"]) # from us
if rc == 0:
raise BMCDisconnected(BMC_DISCONNECT)
rc = console.expect([BMC_DISCONNECT, "\[console-pexpect\]#$"], timeout)
if rc == 0:
raise BMCDisconnected(BMC_DISCONNECT)
exitcode = int(console.before)
print "# LAST COMMAND EXIT CODE %d (%s)" % (exitcode, repr(console.before))
except pexpect.TIMEOUT as e:
print e
print "# TIMEOUT waiting for command to finish."
print "# Attempting to control-c"
try:
console.sendcontrol('c')
rc = console.expect([BMC_DISCONNECT, "\[console-pexpect\]#$"], 10)
if rc == 0:
print "# BMC Disconnect while cancelling timed-out command"
print "# Failing test, disconnecting/reconnecting"
self.terminate()
raise CommandFailed("ipmitool", BMC_DISCONNECT, -1)
if rc == 1:
raise CommandFailed(command, "TIMEOUT", -1)
except pexpect.TIMEOUT:
print "# Timeout trying to kill timed-out command."
print "# Failing current command and attempting to continue"
self.terminate()
raise CommandFailed("ipmitool", "timeout", -1)
raise e
except BMCDisconnected as e:
print "# %s" % str(e)
print "# We can possibly continue..."
print "# Failing current command and attempting to continue"
self.terminate()
self.connect()
console = self.get_console()
print "# On reconnect, attempt to cancel last command (ctrl-c)"
# Note: this is a terrible idea. If BMC vendors created reliable
# SoL implementations this kind of crap wouldn't be needed.
# This is incorrect on so many levels it's not funny. For a start,
# we really don't want to send random control characters during
# boot!
console.sendcontrol('c')
try:
rc = console.expect([BMC_DISCONNECT,
"\[console-pexpect\]#$"], 10)
if rc == 0:
self.terminate()
raise BMCDisconnected(BMC_DISCONNECT)
except pexpect.TIMEOUT:
print "# No response from BMC... trying 'mc reset cold'"
self.mc_reset()
self.terminate()
self.connect()
console = self.get_console()
console.sendcontrol('c')
raise CommandFailed("ipmitool", BMC_DISCONNECT, -1)
if rc == 1:
res = output
res = res.splitlines()
if exitcode != 0:
raise CommandFailed(command, res, exitcode)
return res
else:
res = console.before
res = res.split(command)
return res[-1].splitlines()