def get_dump_dict(quiet=None):
r"""
Get dump information and return as an ordered dictionary where the keys
are the dump IDs and the values are the full path names of the dumps.
Example robot program call:
${dump_dict}= Get Dump Dict
Rpvars 1 dump_dict
Example output:
dump_dict:
[1]:
/var/lib/phosphor-debug-collector/dumps/1/obmcdump_1_1508255216.tar.xz
[2]:
/var/lib/phosphor-debug-collector/dumps/2/obmcdump_2_1508255245.tar.xz
[3]:
/var/lib/phosphor-debug-collector/dumps/3/obmcdump_3_1508255267.tar.xz
[4]:
/var/lib/phosphor-debug-collector/dumps/4/obmcdump_4_1508255283.tar.xz
Description of argument(s):
quiet If quiet is set to 1, this function will
NOT write status messages to stdout.
"""
quiet = int(gp.get_var_value(quiet, 1))
cmd_buf = "dump_dir_path=" + var.DUMP_DIR_PATH + " ; " \
+ "for dump_id in $(ls ${dump_dir_path} | sort -n) ; " \
+ "do echo -n $dump_id: ; ls ${dump_dir_path}${dump_id}/* ; done"
output, stderr, rc = bsu.bmc_execute_command(cmd_buf, quiet=quiet)
return vf.key_value_outbuf_to_dict(output)
def get_dump_dict(quiet=None):
r"""
Get dump information and return as an ordered dictionary where the keys
are the dump IDs and the values are the full path names of the dumps.
Example robot program call:
${dump_dict}= Get Dump Dict
Rprint Vars dump_dict
Example output:
dump_dict:
[1]:
/var/lib/phosphor-debug-collector/dumps/1/obmcdump_1_1508255216.tar.xz
[2]:
/var/lib/phosphor-debug-collector/dumps/2/obmcdump_2_1508255245.tar.xz
[3]:
/var/lib/phosphor-debug-collector/dumps/3/obmcdump_3_1508255267.tar.xz
[4]:
/var/lib/phosphor-debug-collector/dumps/4/obmcdump_4_1508255283.tar.xz
Description of argument(s):
quiet If quiet is set to 1, this function will
NOT write status messages to stdout.
"""
quiet = int(gp.get_var_value(quiet, 1))
cmd_buf = "dump_dir_path=" + var.DUMP_DIR_PATH + " ; " \
+ "for dump_id in $(ls ${dump_dir_path} | sort -n) ; do " \
+ "file_path=$(ls ${dump_dir_path}${dump_id}/* 2>/dev/null)" \
+ " || continue ; echo ${dump_id}:${file_path} ; done"
output, stderr, rc = bsu.bmc_execute_command(cmd_buf, quiet=quiet)
return vf.key_value_outbuf_to_dict(output)
def wait_for_comm_cycle(start_boot_seconds, quiet=None):
r"""
Wait for communications to the BMC to stop working and then resume working.
This function is useful when you have initiated some kind of reboot.
Description of arguments:
start_boot_seconds The time that the boot test started. The format is the
epoch time in seconds, i.e. the number of seconds since
1970-01-01 00:00:00 UTC. This value should be obtained
from the BMC so that it is not dependent on any kind of
synchronization between this machine and the target BMC
This will allow this program to work correctly even in
a simulated environment. This value should be obtained
by the caller prior to initiating a reboot. It can be
obtained as follows:
state = st.get_state(req_states=['epoch_seconds'])
"""
quiet = int(gp.get_var_value(quiet, 0))
# Validate parms.
error_message = gv.svalid_integer(start_boot_seconds,
var_name="start_boot_seconds")
if error_message != "":
BuiltIn().fail(gp.sprint_error(error_message))
match_state = anchor_state(DotDict([('packet_loss', '100')]))
# Wait for 100% packet loss trying to ping machine.
wait_state(match_state, wait_time="8 mins", interval="0 seconds")
match_state['packet_loss'] = '^0$'
# Wait for 0% packet loss trying to ping machine.
wait_state(match_state, wait_time="8 mins", interval="0 seconds")
# Get the uptime and epoch seconds for comparisons. We want to be sure
# that the uptime is less than the elapsed boot time. Further proof that
# a reboot has indeed occurred (vs random network instability giving a
# false positive.
state = get_state(req_states=['uptime', 'epoch_seconds'], quiet=quiet)
elapsed_boot_time = int(state['epoch_seconds']) - start_boot_seconds
gp.qprint_var(elapsed_boot_time)
if int(float(state['uptime'])) < elapsed_boot_time:
uptime = state['uptime']
gp.qprint_var(uptime)
gp.qprint_timen("The uptime is less than the elapsed boot time," +
" as expected.")
else:
error_message = "The uptime is greater than the elapsed boot time," +\
" which is unexpected:\n" +\
gp.sprint_var(start_boot_seconds) +\
gp.sprint_var(state)
BuiltIn().fail(gp.sprint_error(error_message))
gp.qprint_timen("Verifying that REST API interface is working.")
match_state = DotDict([('rest', '^1$')])
state = wait_state(match_state, wait_time="5 mins", interval="2 seconds")
def t_shell_cmd(command_string, **kwargs):
r"""
Search upward in the the call stack to obtain the test_mode argument, add
it to kwargs and then call shell_cmd and return the result.
See shell_cmd prolog for details on all arguments.
"""
if 'test_mode' in kwargs:
error_message = "Programmer error - test_mode is not a valid" +\
" argument to this function."
gp.print_error_report(error_message)
exit(1)
test_mode = gp.get_stack_var('test_mode',
int(gp.get_var_value(None, 0, "test_mode")))
kwargs['test_mode'] = test_mode
return shell_cmd(command_string, **kwargs)
def wait_for_comm_cycle(start_boot_seconds, quiet=None):
r"""
Wait for the BMC uptime to be less than elapsed_boot_time.
This function will tolerate an expected loss of communication to the BMC.
This function is useful when some kind of reboot has been initiated by the
caller.
Description of argument(s):
start_boot_seconds The time that the boot test started. The format is the
epoch time in seconds, i.e. the number of seconds since
1970-01-01 00:00:00 UTC. This value should be obtained
from the BMC so that it is not dependent on any kind of
synchronization between this machine and the target BMC
This will allow this program to work correctly even in
a simulated environment. This value should be obtained
by the caller prior to initiating a reboot. It can be
obtained as follows:
state = st.get_state(req_states=['epoch_seconds'])
"""
quiet = int(gp.get_var_value(quiet, 0))
# Validate parms.
error_message = gv.valid_integer(start_boot_seconds)
if error_message:
BuiltIn().fail(gp.sprint_error(error_message))
# Wait for uptime to be less than elapsed_boot_time.
set_start_boot_seconds(start_boot_seconds)
expr = 'int(float(state[\'uptime\'])) < int(state[\'elapsed_boot_time\'])'
match_state = DotDict([('uptime', '^[0-9\\.]+$'),
('elapsed_boot_time', '^[0-9]+$'),
(expressions_key(), [expr])])
wait_state(match_state, wait_time="12 mins", interval="5 seconds")
gp.qprint_timen("Verifying that REST/Redfish API interface is working.")
if not redfish_support_trans_state:
match_state = DotDict([('rest', '^1$')])
else:
match_state = DotDict([('redfish', '^1$')])
state = wait_state(match_state, wait_time="5 mins", interval="2 seconds")
def check_state(match_state,
invert=0,
print_string="",
openbmc_host="",
openbmc_username="",
openbmc_password="",
os_host="",
os_username="",
os_password="",
quiet=None):
r"""
Check that the Open BMC machine's composite state matches the specified
state. On success, this keyword returns the machine's composite state as a
dictionary.
Description of arguments:
match_state A dictionary whose key/value pairs are "state field"/
"state value". The state value is interpreted as a
regular expression. Example call from robot:
${match_state}= Create Dictionary chassis=^On$
... bmc=^Ready$
... boot_progress=^OSStart$
${state}= Check State &{match_state}
invert If this flag is set, this function will succeed if the
states do NOT match.
print_string This function will print this string to the console prior
to getting the state.
openbmc_host The DNS name or IP address of the BMC.
This defaults to global ${OPENBMC_HOST}.
openbmc_username The username to be used to login to the BMC.
This defaults to global ${OPENBMC_USERNAME}.
openbmc_password The password to be used to login to the BMC.
This defaults to global ${OPENBMC_PASSWORD}.
os_host The DNS name or IP address of the operating system.
This defaults to global ${OS_HOST}.
os_username The username to be used to login to the OS.
This defaults to global ${OS_USERNAME}.
os_password The password to be used to login to the OS.
This defaults to global ${OS_PASSWORD}.
quiet Indicates whether status details should be written to the
console. Defaults to either global value of ${QUIET} or
to 1.
"""
quiet = int(gp.get_var_value(quiet, 0))
grp.rprint(print_string)
req_states = match_state.keys()
# Initialize state.
state = get_state(openbmc_host=openbmc_host,
openbmc_username=openbmc_username,
openbmc_password=openbmc_password,
os_host=os_host,
os_username=os_username,
os_password=os_password,
req_states=req_states,
quiet=quiet)
if not quiet:
gp.print_var(state)
match = compare_states(state, match_state)
if invert and match:
fail_msg = "The current state of the machine matches the match" +\
" state:\n" + gp.sprint_varx("state", state)
BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
elif not invert and not match:
fail_msg = "The current state of the machine does NOT match the" +\
" match state:\n" +\
gp.sprint_varx("state", state)
BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
return state
def shell_cmd(command_string,
quiet=None,
print_output=None,
show_err=1,
test_mode=0,
time_out=None,
max_attempts=1,
retry_sleep_time=5,
allowed_shell_rcs=[0],
ignore_err=None,
return_stderr=0,
fork=0):
r"""
Run the given command string in a shell and return a tuple consisting of
the shell return code and the output.
Description of argument(s):
command_string The command string to be run in a shell
(e.g. "ls /tmp").
quiet If set to 0, this function will print
"Issuing: <cmd string>" to stdout. When
the quiet argument is set to None, this
function will assign a default value by
searching upward in the stack for the
quiet variable value. If no such value is
found, quiet is set to 0.
print_output If this is set, this function will print
the stdout/stderr generated by the shell
command to stdout.
show_err If show_err is set, this function will
print a standardized error report if the
shell command fails (i.e. if the shell
command returns a shell_rc that is not in
allowed_shell_rcs). Note: Error text is
only printed if ALL attempts to run the
command_string fail. In other words, if
the command execution is ultimately
successful, initial failures are hidden.
test_mode If test_mode is set, this function will
not actually run the command. If
print_output is also set, this function
will print "(test_mode) Issuing: <cmd
string>" to stdout. A caller should call
shell_cmd directly if they wish to have
the command string run unconditionally.
They should call the t_shell_cmd wrapper
(defined below) if they wish to run the
command string only if the prevailing
test_mode variable is set to 0.
time_out A time-out value expressed in seconds. If
the command string has not finished
executing within <time_out> seconds, it
will be halted and counted as an error.
max_attempts The max number of attempts that should be
made to run the command string.
retry_sleep_time The number of seconds to sleep between
attempts.
allowed_shell_rcs A list of integers indicating which
shell_rc values are not to be considered
errors.
ignore_err Ignore error means that a failure
encountered by running the command string
will not be raised as a python exception.
When the ignore_err argument is set to
None, this function will assign a default
value by searching upward in the stack for
the ignore_err variable value. If no such
value is found, ignore_err is set to 1.
return_stderr If return_stderr is set, this function
will process the stdout and stderr streams
from the shell command separately. In
such a case, the tuple returned by this
function will consist of three values
rather than just two: rc, stdout, stderr.
fork Run the command string asynchronously
(i.e. don't wait for status of the child
process and don't try to get
stdout/stderr).
"""
# Assign default values to some of the arguments to this function.
quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0)))
print_output = int(gm.dft(print_output, not quiet))
show_err = int(show_err)
global_ignore_err = gp.get_var_value(ignore_err, 1)
stack_ignore_err = gp.get_stack_var('ignore_err', global_ignore_err)
ignore_err = int(gm.dft(ignore_err, gm.dft(stack_ignore_err, 1)))
err_msg = gv.valid_value(command_string)
if err_msg != "":
raise ValueError(err_msg)
if not quiet:
gp.print_issuing(command_string, test_mode)
if test_mode:
if return_stderr:
return 0, "", ""
else:
return 0, ""
# Convert each list entry to a signed value.
allowed_shell_rcs = fa.source_to_object(allowed_shell_rcs)
allowed_shell_rcs = [gm.to_signed(x) for x in allowed_shell_rcs]
if return_stderr:
stderr = subprocess.PIPE
else:
stderr = subprocess.STDOUT
shell_rc = 0
out_buf = ""
err_buf = ""
# Write all output to func_history_stdout rather than directly to stdout.
# This allows us to decide what to print after all attempts to run the
# command string have been made. func_history_stdout will contain the
# complete stdout history from the current invocation of this function.
func_history_stdout = ""
for attempt_num in range(1, max_attempts + 1):
sub_proc = subprocess.Popen(command_string,
bufsize=1,
shell=True,
executable='/bin/bash',
stdout=subprocess.PIPE,
stderr=stderr)
out_buf = ""
err_buf = ""
# Output from this loop iteration is written to func_stdout for later
# processing.
func_stdout = ""
if fork:
break
command_timed_out = False
if time_out is not None:
# Designate a SIGALRM handling function and set alarm.
signal.signal(signal.SIGALRM, shell_cmd_timed_out)
signal.alarm(time_out)
try:
if return_stderr:
for line in sub_proc.stderr:
try:
err_buf += line
except TypeError:
line = line.decode("utf-8")
err_buf += line
if not print_output:
continue
func_stdout += line
for line in sub_proc.stdout:
try:
out_buf += line
except TypeError:
line = line.decode("utf-8")
out_buf += line
if not print_output:
continue
func_stdout += line
except IOError:
command_timed_out = True
sub_proc.communicate()
shell_rc = sub_proc.returncode
# Restore the original SIGALRM handler and clear the alarm.
signal.signal(signal.SIGALRM, original_sigalrm_handler)
signal.alarm(0)
if shell_rc in allowed_shell_rcs:
break
err_msg = "The prior shell command failed.\n"
if quiet:
err_msg += gp.sprint_var(command_string)
if command_timed_out:
err_msg += gp.sprint_var(command_timed_out)
err_msg += gp.sprint_var(time_out)
err_msg += gp.sprint_varx("child_pid", sub_proc.pid)
err_msg += gp.sprint_var(attempt_num)
err_msg += gp.sprint_var(shell_rc, gp.hexa())
err_msg += gp.sprint_var(allowed_shell_rcs, gp.hexa())
if not print_output:
if return_stderr:
err_msg += "err_buf:\n" + err_buf
err_msg += "out_buf:\n" + out_buf
if show_err:
func_stdout += gp.sprint_error_report(err_msg)
func_history_stdout += func_stdout
if attempt_num < max_attempts:
func_history_stdout += gp.sprint_issuing("time.sleep(" +
str(retry_sleep_time) +
")")
time.sleep(retry_sleep_time)
if shell_rc not in allowed_shell_rcs:
func_stdout = func_history_stdout
gp.gp_print(func_stdout)
if shell_rc not in allowed_shell_rcs:
if not ignore_err:
if robot_env:
BuiltIn().fail(err_msg)
else:
raise ValueError("The prior shell command failed.\n")
if return_stderr:
return shell_rc, out_buf, err_buf
else:
return shell_rc, out_buf
def execute_ssh_command(cmd_buf,
open_connection_args={},
login_args={},
print_out=0,
print_err=0,
ignore_err=1,
fork=0,
quiet=None,
test_mode=None):
r"""
Run the given command in an SSH session and return the stdout, stderr and
the return code.
If there is no open SSH connection, this function will connect and login.
Likewise, if the caller has not yet logged in to the connection, this
function will do the login.
NOTE: There is special handling when open_connection_args['alias'] equals
"device_connection".
- A write, rather than an execute_command, is done.
- Only stdout is returned (no stderr or rc).
- print_err, ignore_err and fork are not supported.
Description of arguments:
cmd_buf The command string to be run in an SSH
session.
open_connection_args A dictionary of arg names and values which
are legal to pass to the SSHLibrary
open_connection function as parms/args.
At a minimum, this should contain a 'host'
entry.
login_args A dictionary containing the key/value
pairs which are acceptable to the
SSHLibrary login function as parms/args.
At a minimum, this should contain a
'username' and a 'password' entry.
print_out If this is set, this function will print
the stdout/stderr generated by the shell
command.
print_err If show_err is set, this function will
print a standardized error report if the
shell command returns non-zero.
ignore_err Indicates that errors encountered on the
sshlib.execute_command are to be ignored.
fork Indicates that sshlib.start is to be used
rather than sshlib.execute_command.
quiet Indicates whether this function should run
the pissuing() function which prints an
"Issuing: <cmd string>" to stdout. This
defaults to the global quiet value.
test_mode If test_mode is set, this function will
not actually run the command. This
defaults to the global test_mode value.
"""
gp.lprint_executing()
# Obtain default values.
quiet = int(gp.get_var_value(quiet, 0))
test_mode = int(gp.get_var_value(test_mode, 0))
if not quiet:
gp.pissuing(cmd_buf, test_mode)
gp.lpissuing(cmd_buf, test_mode)
if test_mode:
return "", "", 0
global sshlib
max_exec_cmd_attempts = 2
# Look for existing SSH connection.
# Prepare a search connection dictionary.
search_connection_args = open_connection_args.copy()
# Remove keys that don't work well for searches.
search_connection_args.pop("timeout", None)
connection = find_connection(search_connection_args)
if connection:
gp.lprint_timen("Found the following existing connection:")
gp.lprintn(sprint_connection(connection))
if connection.alias == "":
index_or_alias = connection.index
else:
index_or_alias = connection.alias
gp.lprint_timen("Switching to existing connection: \""
+ str(index_or_alias) + "\".")
sshlib.switch_connection(index_or_alias)
else:
gp.lprint_timen("Connecting to " + open_connection_args['host'] + ".")
cix = sshlib.open_connection(**open_connection_args)
try:
login_ssh(login_args)
except Exception as login_exception:
except_type, except_value, except_traceback = sys.exc_info()
rc = 1
stderr = str(except_value)
stdout = ""
max_exec_cmd_attempts = 0
for exec_cmd_attempt_num in range(1, max_exec_cmd_attempts + 1):
gp.lprint_var(exec_cmd_attempt_num)
try:
if fork:
sshlib.start_command(cmd_buf)
else:
if open_connection_args['alias'] == "device_connection":
stdout = sshlib.write(cmd_buf)
stderr = ""
rc = 0
else:
stdout, stderr, rc = \
sshlib.execute_command(cmd_buf,
return_stdout=True,
return_stderr=True,
return_rc=True)
except Exception as execute_exception:
except_type, except_value, except_traceback = sys.exc_info()
gp.lprint_var(except_type)
gp.lprint_varx("except_value", str(except_value))
if except_type is exceptions.AssertionError and\
re.match(r"Connection not open", str(except_value)):
login_ssh(login_args)
# Now we must continue to next loop iteration to retry the
# execute_command.
continue
if (except_type is paramiko.ssh_exception.SSHException
and re.match(r"SSH session not active", str(except_value))) or\
(except_type is socket.error
and re.match(r"\[Errno 104\] Connection reset by peer",
str(except_value))):
# Close and re-open a connection.
# Note: close_connection() doesn't appear to get rid of the
# connection. It merely closes it. Since there is a concern
# about over-consumption of resources, we use
# close_all_connections() which also gets rid of all
# connections.
gp.lprint_timen("Closing all connections.")
sshlib.close_all_connections()
gp.lprint_timen("Connecting to "
+ open_connection_args['host'] + ".")
cix = sshlib.open_connection(**open_connection_args)
login_ssh(login_args)
continue
# We do not handle any other RuntimeErrors so we will raise the
# exception again.
sshlib.close_all_connections()
raise(execute_exception)
# If we get to this point, the command was executed.
break
if fork:
return
if rc != 0 and print_err:
gp.print_var(rc, 1)
if not print_out:
gp.print_var(stderr)
gp.print_var(stdout)
if print_out:
gp.printn(stderr + stdout)
if not ignore_err:
message = gp.sprint_error("The prior SSH"
+ " command returned a non-zero return"
+ " code:\n" + gp.sprint_var(rc, 1) + stderr
+ "\n")
BuiltIn().should_be_equal(rc, 0, message)
if open_connection_args['alias'] == "device_connection":
return stdout
return stdout, stderr, rc
def execute_ssh_command(cmd_buf,
open_connection_args={},
login_args={},
print_out=0,
print_err=0,
ignore_err=1,
fork=0,
quiet=None,
test_mode=None):
r"""
Run the given command in an SSH session and return the stdout, stderr and
the return code.
If there is no open SSH connection, this function will connect and login.
Likewise, if the caller has not yet logged in to the connection, this
function will do the login.
Description of arguments:
cmd_buf The command string to be run in an SSH
session.
open_connection_args A dictionary of arg names and values which
are legal to pass to the SSHLibrary
open_connection function as parms/args.
At a minimum, this should contain a 'host'
entry.
login_args A dictionary containing the key/value
pairs which are acceptable to the
SSHLibrary login function as parms/args.
At a minimum, this should contain a
'username' and a 'password' entry.
print_out If this is set, this function will print
the stdout/stderr generated by the shell
command.
print_err If show_err is set, this function will
print a standardized error report if the
shell command returns non-zero.
ignore_err Indicates that errors encountered on the
sshlib.execute_command are to be ignored.
fork Indicates that sshlib.start is to be used
rather than sshlib.execute_command.
quiet Indicates whether this function should run
the pissuing() function which prints an
"Issuing: <cmd string>" to stdout. This
defaults to the global quiet value.
test_mode If test_mode is set, this function will
not actually run the command. This
defaults to the global test_mode value.
"""
gp.dprint_executing()
# Obtain default values.
quiet = int(gp.get_var_value(quiet, 0))
test_mode = int(gp.get_var_value(test_mode, 0))
if not quiet:
gp.pissuing(cmd_buf, test_mode)
if test_mode:
return "", "", 0
global sshlib
# Look for existing SSH connection.
# Prepare a search connection dictionary.
search_connection_args = open_connection_args.copy()
# Remove keys that don't work well for searches.
search_connection_args.pop("timeout", None)
connection = find_connection(search_connection_args)
if connection:
gp.dprint_timen("Found the following existing connection:")
gp.dprintn(sprint_connection(connection))
if connection.alias == "":
index_or_alias = connection.index
else:
index_or_alias = connection.alias
gp.dprint_timen("Switching to existing connection: \"" +
str(index_or_alias) + "\".")
sshlib.switch_connection(index_or_alias)
else:
gp.dprint_timen("Connecting to " + open_connection_args['host'] + ".")
cix = sshlib.open_connection(**open_connection_args)
login_ssh(login_args)
max_exec_cmd_attempts = 2
for exec_cmd_attempt_num in range(1, max_exec_cmd_attempts + 1):
gp.dprint_var(exec_cmd_attempt_num)
try:
if fork:
sshlib.start_command(cmd_buf)
else:
stdout, stderr, rc = sshlib.execute_command(cmd_buf,
return_stdout=True,
return_stderr=True,
return_rc=True)
except Exception as execute_exception:
except_type, except_value, except_traceback = sys.exc_info()
gp.dprint_var(except_type)
gp.dprint_varx("except_value", str(except_value))
if except_type is exceptions.AssertionError and\
re.match(r"Connection not open", str(except_value)):
login_ssh(login_args)
# Now we must continue to next loop iteration to retry the
# execute_command.
continue
if except_type is paramiko.ssh_exception.SSHException and\
re.match(r"SSH session not active", str(except_value)):
# Close and re-open a connection.
sshlib.close_connection()
gp.dprint_timen("Connecting to " +
open_connection_args['host'] + ".")
cix = sshlib.open_connection(**open_connection_args)
login_ssh(login_args)
continue
# We do not handle any other RuntimeErrors so we will raise the
# exception again.
raise(execute_exception)
# If we get to this point, the command was executed.
break
if fork:
return
if rc != 0 and print_err:
gp.print_var(rc, 1)
if not print_out:
gp.print_var(stderr)
gp.print_var(stdout)
if print_out:
gp.printn(stderr + stdout)
if not ignore_err:
message = gp.sprint_error("The prior SSH" +
" command returned a non-zero return" +
" code:\n" + gp.sprint_var(rc, 1) + stderr +
"\n")
BuiltIn().should_be_equal(rc, 0, message)
return stdout, stderr, rc
def check_state(match_state,
invert=0,
print_string="",
openbmc_host="",
openbmc_username="",
openbmc_password="",
os_host="",
os_username="",
os_password="",
quiet=None):
r"""
Check that the Open BMC machine's composite state matches the specified
state. On success, this keyword returns the machine's composite state as a
dictionary.
Description of argument(s):
match_state A dictionary whose key/value pairs are "state field"/
"state value". The state value is interpreted as a
regular expression. Example call from robot:
${match_state}= Create Dictionary chassis=^On$
... bmc=^Ready$
... boot_progress=^OSStart$
${state}= Check State &{match_state}
invert If this flag is set, this function will succeed if the
states do NOT match.
print_string This function will print this string to the console prior
to getting the state.
openbmc_host The DNS name or IP address of the BMC.
This defaults to global ${OPENBMC_HOST}.
openbmc_username The username to be used to login to the BMC.
This defaults to global ${OPENBMC_USERNAME}.
openbmc_password The password to be used to login to the BMC.
This defaults to global ${OPENBMC_PASSWORD}.
os_host The DNS name or IP address of the operating system.
This defaults to global ${OS_HOST}.
os_username The username to be used to login to the OS.
This defaults to global ${OS_USERNAME}.
os_password The password to be used to login to the OS.
This defaults to global ${OS_PASSWORD}.
quiet Indicates whether status details should be written to the
console. Defaults to either global value of ${QUIET} or
to 1.
"""
quiet = int(gp.get_var_value(quiet, 0))
gp.gp_print(print_string)
try:
match_state = return_state_constant(match_state)
except TypeError:
pass
req_states = list(match_state.keys())
# Remove special-case match key from req_states.
if expressions_key() in req_states:
req_states.remove(expressions_key())
# Initialize state.
state = get_state(openbmc_host=openbmc_host,
openbmc_username=openbmc_username,
openbmc_password=openbmc_password,
os_host=os_host,
os_username=os_username,
os_password=os_password,
req_states=req_states,
quiet=quiet)
if not quiet:
gp.print_var(state)
if exit_wait_early_message != "":
# The exit_wait_early_message has been set by a signal handler so we
# will exit "successfully". It is incumbent upon the calling function
# (e.g. wait_state) to check/clear this variable and to fail
# appropriately.
return state
match = compare_states(state, match_state)
if invert and match:
fail_msg = "The current state of the machine matches the match" +\
" state:\n" + gp.sprint_varx("state", state)
BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
elif not invert and not match:
fail_msg = "The current state of the machine does NOT match the" +\
" match state:\n" +\
gp.sprint_varx("state", state)
BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
return state
def get_state(openbmc_host="",
openbmc_username="",
openbmc_password="",
os_host="",
os_username="",
os_password="",
req_states=default_req_states,
quiet=None):
r"""
Get component states such as chassis state, bmc state, etc, put them into a
dictionary and return them to the caller.
Note that all substate values are strings.
Description of arguments:
openbmc_host The DNS name or IP address of the BMC.
This defaults to global ${OPENBMC_HOST}.
openbmc_username The username to be used to login to the BMC.
This defaults to global ${OPENBMC_USERNAME}.
openbmc_password The password to be used to login to the BMC.
This defaults to global ${OPENBMC_PASSWORD}.
os_host The DNS name or IP address of the operating system.
This defaults to global ${OS_HOST}.
os_username The username to be used to login to the OS.
This defaults to global ${OS_USERNAME}.
os_password The password to be used to login to the OS.
This defaults to global ${OS_PASSWORD}.
req_states This is a list of states whose values are being requested
by the caller.
quiet Indicates whether status details (e.g. curl commands)
should be written to the console.
Defaults to either global value of ${QUIET} or to 1.
"""
quiet = int(gp.get_var_value(quiet, 0))
# Set parm defaults where necessary and validate all parms.
if openbmc_host == "":
openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}")
error_message = gv.svalid_value(openbmc_host,
var_name="openbmc_host",
invalid_values=[None, ""])
if error_message != "":
BuiltIn().fail(gp.sprint_error(error_message))
if openbmc_username == "":
openbmc_username = BuiltIn().get_variable_value("${OPENBMC_USERNAME}")
error_message = gv.svalid_value(openbmc_username,
var_name="openbmc_username",
invalid_values=[None, ""])
if error_message != "":
BuiltIn().fail(gp.sprint_error(error_message))
if openbmc_password == "":
openbmc_password = BuiltIn().get_variable_value("${OPENBMC_PASSWORD}")
error_message = gv.svalid_value(openbmc_password,
var_name="openbmc_password",
invalid_values=[None, ""])
if error_message != "":
BuiltIn().fail(gp.sprint_error(error_message))
# NOTE: OS parms are optional.
if os_host == "":
os_host = BuiltIn().get_variable_value("${OS_HOST}")
if os_host is None:
os_host = ""
if os_username is "":
os_username = BuiltIn().get_variable_value("${OS_USERNAME}")
if os_username is None:
os_username = ""
if os_password is "":
os_password = BuiltIn().get_variable_value("${OS_PASSWORD}")
if os_password is None:
os_password = ""
invalid_req_states = [
sub_state for sub_state in req_states
if sub_state not in valid_req_states
]
if len(invalid_req_states) > 0:
error_message = "The following req_states are not supported:\n" +\
gp.sprint_var(invalid_req_states)
BuiltIn().fail(gp.sprint_error(error_message))
# Initialize all substate values supported by this function.
ping = 0
packet_loss = ''
uptime = ''
epoch_seconds = ''
rest = '1'
chassis = ''
bmc = ''
boot_progress = ''
host = ''
# Get the component states.
if 'ping' in req_states:
# See if the OS pings.
cmd_buf = "ping -c 1 -w 2 " + openbmc_host
if not quiet:
gp.pissuing(cmd_buf)
rc, out_buf = commands.getstatusoutput(cmd_buf)
if rc == 0:
ping = 1
if 'packet_loss' in req_states:
# See if the OS pings.
cmd_buf = "ping -c 5 -w 5 " + openbmc_host +\
" | egrep 'packet loss' | sed -re 's/.* ([0-9]+)%.*/\\1/g'"
if not quiet:
gp.pissuing(cmd_buf)
rc, out_buf = commands.getstatusoutput(cmd_buf)
if rc == 0:
packet_loss = out_buf.rstrip("\n")
if 'uptime' in req_states:
cmd_buf = [
"BMC Execute Command", "cat /proc/uptime | cut -f 1 -d ' '",
'quiet=${1}'
]
if not quiet:
grp.rpissuing_keyword(cmd_buf)
status, ret_values = \
BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
if status == "PASS":
stdout, stderr, rc = ret_values
if rc == 0 and stderr == "":
uptime = stdout
if 'epoch_seconds' in req_states:
date_cmd_buf = "date -u +%s"
if USE_BMC_EPOCH_TIME:
cmd_buf = ["BMC Execute Command", date_cmd_buf, 'quiet=${1}']
if not quiet:
grp.rpissuing_keyword(cmd_buf)
status, ret_values = \
BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
if status == "PASS":
stdout, stderr, rc = ret_values
if rc == 0 and stderr == "":
epoch_seconds = stdout.rstrip("\n")
else:
shell_rc, out_buf = gc.cmd_fnc_u(date_cmd_buf,
quiet=1,
print_output=0)
if shell_rc == 0:
epoch_seconds = out_buf.rstrip("\n")
master_req_rest = ['rest', 'chassis', 'bmc', 'boot_progress', 'host']
req_rest = [
sub_state for sub_state in req_states if sub_state in master_req_rest
]
need_rest = (len(req_rest) > 0)
# Though we could try to determine 'rest' state on any of several calls,
# for simplicity, we'll use 'chassis' to figure it out (even if the caller
# hasn't explicitly asked for 'chassis').
if 'chassis' in req_states or need_rest:
cmd_buf = ["Get Chassis Power State", "quiet=${" + str(quiet) + "}"]
grp.rdpissuing_keyword(cmd_buf)
status, ret_values = \
BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
if status == "PASS":
chassis = ret_values
chassis = re.sub(r'.*\.', "", chassis)
rest = '1'
else:
rest = ret_values
if rest == '1':
if 'bmc' in req_states:
if OBMC_STATES_VERSION == 0:
qualifier = "utils"
else:
# This will not be supported much longer.
qualifier = "state_manager"
cmd_buf = [
qualifier + ".Get BMC State", "quiet=${" + str(quiet) + "}"
]
grp.rdpissuing_keyword(cmd_buf)
status, ret_values = \
BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
if status == "PASS":
bmc = ret_values
if 'boot_progress' in req_states:
cmd_buf = ["Get Boot Progress", "quiet=${" + str(quiet) + "}"]
grp.rdpissuing_keyword(cmd_buf)
status, ret_values = \
BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
if status == "PASS":
boot_progress = ret_values
if 'host' in req_states:
if OBMC_STATES_VERSION > 0:
cmd_buf = ["Get Host State", "quiet=${" + str(quiet) + "}"]
grp.rdpissuing_keyword(cmd_buf)
status, ret_values = \
BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
if status == "PASS":
host = ret_values
# Strip everything up to the final period.
host = re.sub(r'.*\.', "", host)
state = DotDict()
for sub_state in req_states:
if sub_state.startswith("os_"):
# We pass "os_" requests on to get_os_state.
continue
cmd_buf = "state['" + sub_state + "'] = str(" + sub_state + ")"
exec(cmd_buf)
if os_host == "":
# The caller has not specified an os_host so as far as we're concerned,
# it doesn't exist.
return state
os_req_states = [
sub_state for sub_state in req_states if sub_state.startswith('os_')
]
if len(os_req_states) > 0:
# The caller has specified an os_host and they have requested
# information on os substates.
# Based on the information gathered on bmc, we'll try to make a
# determination of whether the os is even up. We'll pass the result
# of that assessment to get_os_state to enhance performance.
os_up_match = DotDict()
for sub_state in master_os_up_match:
if sub_state in req_states:
os_up_match[sub_state] = master_os_up_match[sub_state]
os_up = compare_states(state, os_up_match)
os_state = get_os_state(os_host=os_host,
os_username=os_username,
os_password=os_password,
req_states=os_req_states,
os_up=os_up,
quiet=quiet)
# Append os_state dictionary to ours.
state.update(os_state)
return state
def get_os_state(os_host="",
os_username="",
os_password="",
req_states=default_os_req_states,
os_up=True,
quiet=None):
r"""
Get component states for the operating system such as ping, login,
etc, put them into a dictionary and return them to the caller.
Note that all substate values are strings.
Description of arguments:
os_host The DNS name or IP address of the operating system.
This defaults to global ${OS_HOST}.
os_username The username to be used to login to the OS.
This defaults to global ${OS_USERNAME}.
os_password The password to be used to login to the OS.
This defaults to global ${OS_PASSWORD}.
req_states This is a list of states whose values are being requested by
the caller.
os_up If the caller knows that the os can't possibly be up, it can
improve performance by passing os_up=False. This function
will then simply return default values for all requested os
sub states.
quiet Indicates whether status details (e.g. curl commands) should
be written to the console.
Defaults to either global value of ${QUIET} or to 1.
"""
quiet = int(gp.get_var_value(quiet, 0))
# Set parm defaults where necessary and validate all parms.
if os_host == "":
os_host = BuiltIn().get_variable_value("${OS_HOST}")
error_message = gv.svalid_value(os_host,
var_name="os_host",
invalid_values=[None, ""])
if error_message != "":
BuiltIn().fail(gp.sprint_error(error_message))
if os_username == "":
os_username = BuiltIn().get_variable_value("${OS_USERNAME}")
error_message = gv.svalid_value(os_username,
var_name="os_username",
invalid_values=[None, ""])
if error_message != "":
BuiltIn().fail(gp.sprint_error(error_message))
if os_password == "":
os_password = BuiltIn().get_variable_value("${OS_PASSWORD}")
error_message = gv.svalid_value(os_password,
var_name="os_password",
invalid_values=[None, ""])
if error_message != "":
BuiltIn().fail(gp.sprint_error(error_message))
invalid_req_states = [
sub_state for sub_state in req_states
if sub_state not in valid_os_req_states
]
if len(invalid_req_states) > 0:
error_message = "The following req_states are not supported:\n" +\
gp.sprint_var(invalid_req_states)
BuiltIn().fail(gp.sprint_error(error_message))
# Initialize all substate values supported by this function.
os_ping = 0
os_login = 0
os_run_cmd = 0
if os_up:
if 'os_ping' in req_states:
# See if the OS pings.
cmd_buf = "ping -c 1 -w 2 " + os_host
if not quiet:
gp.pissuing(cmd_buf)
rc, out_buf = commands.getstatusoutput(cmd_buf)
if rc == 0:
os_ping = 1
# Programming note: All attributes which do not require an ssh login
# should have been processed by this point.
master_req_login = ['os_login', 'os_run_cmd']
req_login = [
sub_state for sub_state in req_states
if sub_state in master_req_login
]
must_login = (len(req_login) > 0)
if must_login:
# Open SSH connection to OS. Note that this doesn't fail even when
# the OS is not up.
cmd_buf = ["SSHLibrary.Open Connection", os_host]
if not quiet:
grp.rpissuing_keyword(cmd_buf)
ix = BuiltIn().run_keyword(*cmd_buf)
# Login to OS.
cmd_buf = ["Login", os_username, os_password]
if not quiet:
grp.rpissuing_keyword(cmd_buf)
status, ret_values = \
BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
if status == "PASS":
os_login = 1
else:
gp.dprint_var(status)
gp.dprint_var(ret_values)
if os_login:
if 'os_run_cmd' in req_states:
# Try running a simple command (uptime) on the OS.
cmd_buf = [
"Execute Command", "uptime", "return_stderr=True",
"return_rc=True"
]
if not quiet:
grp.rpissuing_keyword(cmd_buf)
# Note that in spite of its name, there are occasions
# where run_keyword_and_ignore_error can fail.
status, ret_values = \
BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
if status == "PASS":
stdout, stderr, rc = ret_values
if rc == 0 and stderr == "":
os_run_cmd = 1
else:
gp.dprint_var(status)
gp.dprint_var(stdout)
gp.dprint_var(stderr)
gp.dprint_var(rc)
else:
gp.dprint_var(status)
gp.dprint_var(ret_values)
os_state = DotDict()
for sub_state in req_states:
cmd_buf = "os_state['" + sub_state + "'] = str(" + sub_state + ")"
exec(cmd_buf)
return os_state
def build_ipmi_ext_cmd(quiet=None):
r"""
Build the global IPMI_EXT_CMD variable.
If global variable IPMI_EXT_CMD already has a value, this keyword will
simply return without taking any action.
This keyword is designed for use by keywords which use the IPMI_EXT_CMD
variable (e.g. 'Run External IPMI Raw Command'). This keyword is
warranted because the ipmitool program may or may not accept the -U (i.e.
username) parameter depending on the version of code loaded on the BMC.
This keyword will determine whether the "-U" parameter should be used and
create IPMI_EXT_CMD accordingly.
Furthermore, this keyword will run the command to create the 'root' IPMI
username.
Description of argument(s):
# quiet Indicates whether this keyword should run
# without any output to the console.
"""
ipmi_ext_cmd = BuiltIn().get_variable_value("${IPMI_EXT_CMD}", "")
if ipmi_ext_cmd != "":
return
quiet = int(gp.get_var_value(quiet, 0))
openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}")
ipmi_username = BuiltIn().get_variable_value("${IPMI_USERNAME}", "root")
ipmi_password = BuiltIn().get_variable_value("${IPMI_PASSWORD}", "0penBmc")
ipmi_cipher_level = BuiltIn().get_variable_value("${IPMI_CIPHER_LEVEL}",
"3")
old_ipmi_ext_cmd = "ipmitool -I lanplus -C " + str(ipmi_cipher_level)\
+ " -P " + ipmi_password
new_ipmi_ext_cmd = "ipmitool -I lanplus -C " + str(ipmi_cipher_level)\
+ " -U " + ipmi_username + " -P " + ipmi_password
# Use a basic ipmitool command to help us determine whether the BMC will
# accept the -U parm.
ipmi_cmd = "power status"
ipmi_cmd_suffix = " -H " + openbmc_host + " " + ipmi_cmd
print_output = 0
cmd_buf = new_ipmi_ext_cmd + ipmi_cmd_suffix
new_rc, stdout = gc.shell_cmd(cmd_buf,
print_output=print_output,
show_err=0,
ignore_err=1)
gp.qprint_varx("rc", new_rc, 1)
if new_rc == 0:
ipmi_ext_cmd = new_ipmi_ext_cmd
BuiltIn().set_global_variable("${IPMI_EXT_CMD}", ipmi_ext_cmd)
return
cmd_buf = old_ipmi_ext_cmd + ipmi_cmd_suffix
old_rc, stdout = gc.shell_cmd(cmd_buf,
print_output=print_output,
show_err=0,
ignore_err=1)
gp.qprint_varx("rc", old_rc, 1)
if old_rc == 0:
ipmi_ext_cmd = old_ipmi_ext_cmd
BuiltIn().set_global_variable("${IPMI_EXT_CMD}", ipmi_ext_cmd)
return
message = "Unable to run ipmitool, (with or without the '-U' parm)."
BuiltIn().fail(message)
def execute_ssh_command(cmd_buf,
open_connection_args={},
login_args={},
print_out=0,
print_err=0,
ignore_err=1,
fork=0,
quiet=None,
test_mode=None,
time_out=None):
r"""
Run the given command in an SSH session and return the stdout, stderr and
the return code.
If there is no open SSH connection, this function will connect and login.
Likewise, if the caller has not yet logged in to the connection, this
function will do the login.
NOTE: There is special handling when open_connection_args['alias'] equals
"device_connection".
- A write, rather than an execute_command, is done.
- Only stdout is returned (no stderr or rc).
- print_err, ignore_err and fork are not supported.
Description of arguments:
cmd_buf The command string to be run in an SSH
session.
open_connection_args A dictionary of arg names and values which
are legal to pass to the SSHLibrary
open_connection function as parms/args.
At a minimum, this should contain a 'host'
entry.
login_args A dictionary containing the key/value
pairs which are acceptable to the
SSHLibrary login function as parms/args.
At a minimum, this should contain a
'username' and a 'password' entry.
print_out If this is set, this function will print
the stdout/stderr generated by the shell
command.
print_err If show_err is set, this function will
print a standardized error report if the
shell command returns non-zero.
ignore_err Indicates that errors encountered on the
sshlib.execute_command are to be ignored.
fork Indicates that sshlib.start is to be used
rather than sshlib.execute_command.
quiet Indicates whether this function should run
the pissuing() function which prints an
"Issuing: <cmd string>" to stdout. This
defaults to the global quiet value.
test_mode If test_mode is set, this function will
not actually run the command. This
defaults to the global test_mode value.
time_out The amount of time to allow for the
execution of cmd_buf. A value of None
means that there is no limit to how long
the command may take.
"""
gp.lprint_executing()
# Obtain default values.
quiet = int(gp.get_var_value(quiet, 0))
test_mode = int(gp.get_var_value(test_mode, 0))
if not quiet:
gp.pissuing(cmd_buf, test_mode)
gp.lpissuing(cmd_buf, test_mode)
if test_mode:
return "", "", 0
global sshlib
max_exec_cmd_attempts = 2
# Look for existing SSH connection.
# Prepare a search connection dictionary.
search_connection_args = open_connection_args.copy()
# Remove keys that don't work well for searches.
search_connection_args.pop("timeout", None)
connection = find_connection(search_connection_args)
if connection:
gp.lprint_timen("Found the following existing connection:")
gp.lprintn(sprint_connection(connection))
if connection.alias == "":
index_or_alias = connection.index
else:
index_or_alias = connection.alias
gp.lprint_timen("Switching to existing connection: \"" +
str(index_or_alias) + "\".")
sshlib.switch_connection(index_or_alias)
else:
gp.lprint_timen("Connecting to " + open_connection_args['host'] + ".")
cix = sshlib.open_connection(**open_connection_args)
try:
login_ssh(login_args)
except Exception:
except_type, except_value, except_traceback = sys.exc_info()
rc = 1
stderr = str(except_value)
stdout = ""
max_exec_cmd_attempts = 0
for exec_cmd_attempt_num in range(1, max_exec_cmd_attempts + 1):
gp.lprint_var(exec_cmd_attempt_num)
try:
if fork:
sshlib.start_command(cmd_buf)
else:
if open_connection_args['alias'] == "device_connection":
stdout = sshlib.write(cmd_buf)
stderr = ""
rc = 0
else:
stdout, stderr, rc = \
func_timer.run(sshlib.execute_command,
cmd_buf,
return_stdout=True,
return_stderr=True,
return_rc=True,
time_out=time_out)
except Exception:
except_type, except_value, except_traceback = sys.exc_info()
gp.lprint_var(except_type)
gp.lprint_varx("except_value", str(except_value))
# This may be our last time through the retry loop, so setting
# return variables.
rc = 1
stderr = str(except_value)
stdout = ""
if except_type is exceptions.AssertionError and\
re.match(r"Connection not open", str(except_value)):
try:
login_ssh(login_args)
# Now we must continue to next loop iteration to retry the
# execute_command.
continue
except Exception:
except_type, except_value, except_traceback =\
sys.exc_info()
rc = 1
stderr = str(except_value)
stdout = ""
break
if (except_type is paramiko.ssh_exception.SSHException
and re.match(r"SSH session not active", str(except_value))) or\
(except_type is socket.error
and re.match(r"\[Errno 104\] Connection reset by peer",
str(except_value))) or\
(except_type is paramiko.ssh_exception.SSHException
and re.match(r"Timeout opening channel\.",
str(except_value))):
# Close and re-open a connection.
# Note: close_connection() doesn't appear to get rid of the
# connection. It merely closes it. Since there is a concern
# about over-consumption of resources, we use
# close_all_connections() which also gets rid of all
# connections.
gp.lprint_timen("Closing all connections.")
sshlib.close_all_connections()
gp.lprint_timen("Connecting to " +
open_connection_args['host'] + ".")
cix = sshlib.open_connection(**open_connection_args)
login_ssh(login_args)
continue
# We do not handle any other RuntimeErrors so we will raise the
# exception again.
sshlib.close_all_connections()
gp.lprintn(traceback.format_exc())
raise (except_value)
# If we get to this point, the command was executed.
break
if fork:
return
if rc != 0 and print_err:
gp.print_var(rc, gp.hexa())
if not print_out:
gp.print_var(stderr)
gp.print_var(stdout)
if print_out:
gp.printn(stderr + stdout)
if not ignore_err:
message = gp.sprint_error("The prior SSH" +
" command returned a non-zero return" +
" code:\n" + gp.sprint_var(rc, gp.hexa()) +
stderr + "\n")
BuiltIn().should_be_equal(rc, 0, message)
if open_connection_args['alias'] == "device_connection":
return stdout
return stdout, stderr, rc
def run_key(keyword_buf,
quiet=None,
test_mode=None,
ignore=0):
r"""
Run the given keyword, return the status and the keyword return values.
The advantage of using this function verses having robot simply run your
keyword is the handling of parameters like quiet, test_mode and ignore.
Description of arguments:
keyword_buf The keyword string to be run.
quiet Indicates whether this function should run
the pissuing function to print 'Issuing:
<keyword string>' to stdout.
test_mode If test_mode is set, this function will
not actually run the command. If quiet is
0, it will print a message indicating what
it would have run (e.g. "Issuing:
(test_mode) your command").
ignore Ignore errors from running keyword. If
this is 0, this function will fail with
whatever error occurred when running the
keyword.
Example usage from a robot script:
${status} ${ret_values}= Run Key My Keyword \ Arg1 \ Arg2
Note that to get robot to pass your command + args as a single string to
this function, you must escape extra spaces with a backslash.
Also note that ret_values is a python list:
ret_values:
ret_values[0]: value1
ret_values[1]: value2
"""
# Set these vars to default values if they are None.
quiet = int(gp.get_var_value(quiet, 0))
test_mode = int(gp.get_var_value(test_mode, 0))
ignore = int(ignore)
# Convert the keyword_buf into a list split wherever 2 or more spaces are
# found.
keyword_list = keyword_buf.split(' ')
# Strip spaces from each argument to make the output look clean and
# uniform.
keyword_list = [item.strip(' ') for item in keyword_list]
if not quiet:
# Join the list back into keyword_buf for the sake of output.
keyword_buf = ' '.join(keyword_list)
gp.pissuing(keyword_buf, test_mode)
if test_mode:
return 'PASS', ""
try:
status, ret_values = \
BuiltIn().run_keyword_and_ignore_error(*keyword_list)
except Exception as my_assertion_error:
status = "FAIL"
ret_values = my_assertion_error.args[0]
if status != 'PASS':
# Output the error message to stderr.
BuiltIn().log_to_console(ret_values, stream='STDERR')
if not ignore:
# Fail with the given error message.
BuiltIn().fail(ret_values)
return status, ret_values
def get_os_state(os_host="",
os_username="",
os_password="",
req_states=default_os_req_states,
os_up=True,
quiet=None):
r"""
Get component states for the operating system such as ping, login,
etc, put them into a dictionary and return them to the caller.
Note that all substate values are strings.
Description of argument(s):
os_host The DNS name or IP address of the operating system.
This defaults to global ${OS_HOST}.
os_username The username to be used to login to the OS.
This defaults to global ${OS_USERNAME}.
os_password The password to be used to login to the OS.
This defaults to global ${OS_PASSWORD}.
req_states This is a list of states whose values are being requested by
the caller.
os_up If the caller knows that the os can't possibly be up, it can
improve performance by passing os_up=False. This function
will then simply return default values for all requested os
sub states.
quiet Indicates whether status details (e.g. curl commands) should
be written to the console.
Defaults to either global value of ${QUIET} or to 1.
"""
quiet = int(gp.get_var_value(quiet, 0))
# Set parm defaults where necessary and validate all parms.
if os_host == "":
os_host = BuiltIn().get_variable_value("${OS_HOST}")
error_message = gv.valid_value(os_host, invalid_values=[None, ""])
if error_message != "":
BuiltIn().fail(gp.sprint_error(error_message))
if os_username == "":
os_username = BuiltIn().get_variable_value("${OS_USERNAME}")
error_message = gv.valid_value(os_username, invalid_values=[None, ""])
if error_message != "":
BuiltIn().fail(gp.sprint_error(error_message))
if os_password == "":
os_password = BuiltIn().get_variable_value("${OS_PASSWORD}")
error_message = gv.valid_value(os_password, invalid_values=[None, ""])
if error_message != "":
BuiltIn().fail(gp.sprint_error(error_message))
invalid_req_states = [sub_state for sub_state in req_states
if sub_state not in valid_os_req_states]
if len(invalid_req_states) > 0:
error_message = "The following req_states are not supported:\n" +\
gp.sprint_var(invalid_req_states)
BuiltIn().fail(gp.sprint_error(error_message))
# Initialize all substate values supported by this function.
os_ping = 0
os_login = 0
os_run_cmd = 0
if os_up:
if 'os_ping' in req_states:
# See if the OS pings.
rc, out_buf = gc.shell_cmd("ping -c 1 -w 2 " + os_host,
print_output=0, show_err=0,
ignore_err=1)
if rc == 0:
os_ping = 1
# Programming note: All attributes which do not require an ssh login
# should have been processed by this point.
master_req_login = ['os_login', 'os_run_cmd']
req_login = [sub_state for sub_state in req_states if sub_state in
master_req_login]
must_login = (len(req_login) > 0)
if must_login:
output, stderr, rc = bsu.os_execute_command("uptime", quiet=quiet,
ignore_err=1,
time_out=20)
if rc == 0:
os_login = 1
os_run_cmd = 1
else:
gp.dprint_vars(output, stderr)
gp.dprint_vars(rc, 1)
os_state = DotDict()
for sub_state in req_states:
cmd_buf = "os_state['" + sub_state + "'] = str(" + sub_state + ")"
exec(cmd_buf)
return os_state
def get_state(openbmc_host="",
openbmc_username="",
openbmc_password="",
os_host="",
os_username="",
os_password="",
req_states=default_req_states,
quiet=None):
r"""
Get component states such as chassis state, bmc state, etc, put them into a
dictionary and return them to the caller.
Note that all substate values are strings.
Note: If elapsed_boot_time is included in req_states, it is the caller's
duty to call set_start_boot_seconds() in order to set global
start_boot_seconds. elapsed_boot_time is the current time minus
start_boot_seconds.
Description of argument(s):
openbmc_host The DNS name or IP address of the BMC.
This defaults to global ${OPENBMC_HOST}.
openbmc_username The username to be used to login to the BMC.
This defaults to global ${OPENBMC_USERNAME}.
openbmc_password The password to be used to login to the BMC.
This defaults to global ${OPENBMC_PASSWORD}.
os_host The DNS name or IP address of the operating system.
This defaults to global ${OS_HOST}.
os_username The username to be used to login to the OS.
This defaults to global ${OS_USERNAME}.
os_password The password to be used to login to the OS.
This defaults to global ${OS_PASSWORD}.
req_states This is a list of states whose values are being requested
by the caller.
quiet Indicates whether status details (e.g. curl commands)
should be written to the console.
Defaults to either global value of ${QUIET} or to 1.
"""
quiet = int(gp.get_var_value(quiet, 0))
# Set parm defaults where necessary and validate all parms.
if openbmc_host == "":
openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}")
error_message = gv.valid_value(openbmc_host, invalid_values=[None, ""])
if error_message != "":
BuiltIn().fail(gp.sprint_error(error_message))
if openbmc_username == "":
openbmc_username = BuiltIn().get_variable_value("${OPENBMC_USERNAME}")
error_message = gv.valid_value(openbmc_username, invalid_values=[None, ""])
if error_message != "":
BuiltIn().fail(gp.sprint_error(error_message))
if openbmc_password == "":
openbmc_password = BuiltIn().get_variable_value("${OPENBMC_PASSWORD}")
error_message = gv.valid_value(openbmc_password, invalid_values=[None, ""])
if error_message != "":
BuiltIn().fail(gp.sprint_error(error_message))
# NOTE: OS parms are optional.
if os_host == "":
os_host = BuiltIn().get_variable_value("${OS_HOST}")
if os_host is None:
os_host = ""
if os_username is "":
os_username = BuiltIn().get_variable_value("${OS_USERNAME}")
if os_username is None:
os_username = ""
if os_password is "":
os_password = BuiltIn().get_variable_value("${OS_PASSWORD}")
if os_password is None:
os_password = ""
invalid_req_states = [sub_state for sub_state in req_states
if sub_state not in valid_req_states]
if len(invalid_req_states) > 0:
error_message = "The following req_states are not supported:\n" +\
gp.sprint_var(invalid_req_states)
BuiltIn().fail(gp.sprint_error(error_message))
# Initialize all substate values supported by this function.
ping = 0
packet_loss = ''
uptime = ''
epoch_seconds = ''
elapsed_boot_time = ''
rest = ''
chassis = ''
requested_chassis = ''
bmc = ''
requested_bmc = ''
boot_progress = ''
operating_system = ''
host = ''
requested_host = ''
attempts_left = ''
# Get the component states.
if 'ping' in req_states:
# See if the OS pings.
rc, out_buf = gc.shell_cmd("ping -c 1 -w 2 " + openbmc_host,
print_output=0, show_err=0,
ignore_err=1)
if rc == 0:
ping = 1
if 'packet_loss' in req_states:
# See if the OS pings.
cmd_buf = "ping -c 5 -w 5 " + openbmc_host +\
" | egrep 'packet loss' | sed -re 's/.* ([0-9]+)%.*/\\1/g'"
rc, out_buf = gc.shell_cmd(cmd_buf,
print_output=0, show_err=0,
ignore_err=1)
if rc == 0:
packet_loss = out_buf.rstrip("\n")
if 'uptime' in req_states:
# Sometimes reading uptime results in a blank value. Call with
# wait_until_keyword_succeeds to ensure a non-blank value is obtained.
remote_cmd_buf = "read uptime filler 2>/dev/null < /proc/uptime" +\
" && [ ! -z \"${uptime}\" ] && echo ${uptime}"
cmd_buf = ["BMC Execute Command",
re.sub('\\$', '\\$', remote_cmd_buf), 'quiet=1',
'test_mode=0']
gp.qprint_issuing(cmd_buf, 0)
gp.qprint_issuing(remote_cmd_buf, 0)
try:
stdout, stderr, rc =\
BuiltIn().wait_until_keyword_succeeds("10 sec", "0 sec",
*cmd_buf)
if rc == 0 and stderr == "":
uptime = stdout
except AssertionError as my_assertion_error:
pass
if 'epoch_seconds' in req_states or 'elapsed_boot_time' in req_states:
date_cmd_buf = "date -u +%s"
if USE_BMC_EPOCH_TIME:
cmd_buf = ["BMC Execute Command", date_cmd_buf, 'quiet=${1}']
if not quiet:
gp.print_issuing(cmd_buf)
status, ret_values = \
BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
if status == "PASS":
stdout, stderr, rc = ret_values
if rc == 0 and stderr == "":
epoch_seconds = stdout.rstrip("\n")
else:
shell_rc, out_buf = gc.cmd_fnc_u(date_cmd_buf,
quiet=quiet,
print_output=0)
if shell_rc == 0:
epoch_seconds = out_buf.rstrip("\n")
if 'elapsed_boot_time' in req_states:
global start_boot_seconds
elapsed_boot_time = int(epoch_seconds) - start_boot_seconds
master_req_rest = ['rest', 'host', 'requested_host', 'operating_system',
'attempts_left', 'boot_progress', 'chassis',
'requested_chassis' 'bmc' 'requested_bmc']
req_rest = [sub_state for sub_state in req_states if sub_state in
master_req_rest]
need_rest = (len(req_rest) > 0)
state = DotDict()
if need_rest:
cmd_buf = ["Read Properties", SYSTEM_STATE_URI + "enumerate",
"quiet=${" + str(quiet) + "}"]
gp.dprint_issuing(cmd_buf)
status, ret_values = \
BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
if status == "PASS":
state['rest'] = '1'
else:
state['rest'] = '0'
if int(state['rest']):
for url_path in ret_values:
for attr_name in ret_values[url_path]:
# Create a state key value based on the attr_name.
try:
ret_values[url_path][attr_name] = \
re.sub(r'.*\.', "",
ret_values[url_path][attr_name])
except TypeError:
pass
# Do some key name manipulations.
new_attr_name = re.sub(r'^Current|(State|Transition)$',
"", attr_name)
new_attr_name = re.sub(r'BMC', r'Bmc', new_attr_name)
new_attr_name = re.sub(r'([A-Z][a-z])', r'_\1',
new_attr_name)
new_attr_name = new_attr_name.lower().lstrip("_")
new_attr_name = re.sub(r'power', r'chassis', new_attr_name)
if new_attr_name in req_states:
state[new_attr_name] = ret_values[url_path][attr_name]
for sub_state in req_states:
if sub_state in state:
continue
if sub_state.startswith("os_"):
# We pass "os_" requests on to get_os_state.
continue
cmd_buf = "state['" + sub_state + "'] = str(" + sub_state + ")"
exec(cmd_buf)
if os_host == "":
# The caller has not specified an os_host so as far as we're concerned,
# it doesn't exist.
return state
os_req_states = [sub_state for sub_state in req_states
if sub_state.startswith('os_')]
if len(os_req_states) > 0:
# The caller has specified an os_host and they have requested
# information on os substates.
# Based on the information gathered on bmc, we'll try to make a
# determination of whether the os is even up. We'll pass the result
# of that assessment to get_os_state to enhance performance.
os_up_match = DotDict()
for sub_state in master_os_up_match:
if sub_state in req_states:
os_up_match[sub_state] = master_os_up_match[sub_state]
os_up = compare_states(state, os_up_match)
os_state = get_os_state(os_host=os_host,
os_username=os_username,
os_password=os_password,
req_states=os_req_states,
os_up=os_up,
quiet=quiet)
# Append os_state dictionary to ours.
state.update(os_state)
return state
def rprocess_plug_in_packages(plug_in_packages_list=None,
call_point="setup",
shell_rc="0x00000000",
stop_on_plug_in_failure=1,
stop_on_non_zero_rc=0,
release_type="obmc",
quiet=None,
debug=None,
return_history=False):
r"""
Call the external process_plug_in_packages.py to process the plug-in packages. Return the following:
rc The return code - 0 = PASS, 1 = FAIL.
shell_rc The shell return code returned by process_plug_in_packages.py.
failed_plug_in_name The failed plug in name (if any).
Description of arguments:
plug_in_packages_list A python list of plug-in directory paths.
call_point The call point program to be called for each plug-in package (e.g.
post_boot). This name should not include the "cp_" prefix.
shell_rc The user may supply a value other than zero to indicate an acceptable
non-zero return code. For example, if this value equals 0x00000200, it
means that for each plug-in call point that runs, a 0x00000200 will not
be counted as a failure.
stop_on_plug_in_failure If this parameter is set to 1, this program will stop and return non-zero
if the call point program from any plug-in directory fails. Conversely,
if it is set to false, this program will run the call point program from
each and every plug-in directory regardless of their return values.
Typical example cases where you'd want to run all plug-in call points
regardless of success or failure would be "cleanup" or "ffdc" call points.
stop_on_non_zero_rc If this parm is set to 1 and a plug-in call point program returns a valid
non-zero return code (see "shell_rc" parm above), this program will stop
processing and return 0 (success). Since this constitutes a successful
exit, this would normally be used where the caller wishes to stop
processing if one of the plug-in directory call point programs returns a
special value indicating that some special case has been found. An
example might be in calling some kind of "check_errl" call point program.
Such a call point program might return a 2 (i.e. 0x00000200) to indicate
that a given error log entry was found in an "ignore" list and is
therefore to be ignored. That being the case, no other "check_errl" call
point program would need to be called.
release_type The type of release being tested (e.g. "obmc", "op", "fips"). This
influences which integrated plug-ins are selected.
quiet If quiet is set to 1, this function will NOT write status messages to
stdout. This will default to the global quiet program parm or to 0.
debug If this parameter is set to 1, this function will print additional debug
information. This is mainly to be used by the developer of this
function. This will default to the global quiet program parm or to 0.
return_history In addition to rc, shell_rc and failed_plug_in_name, return a list
containing historical output that looks like the following:
history:
history[0]: #(CDT) 2018/10/30 12:25:49 - Running OBMC_Sample/cp_post_stack
"""
rc = 0
plug_in_packages_list = gp.get_var_value(plug_in_packages_list, [])
# If there are no plug-in packages to process, return successfully.
if len(plug_in_packages_list) == 0:
if return_history:
return 0, 0, "", []
else:
return 0, 0, ""
quiet = int(gp.get_var_value(quiet, 0))
debug = int(gp.get_var_value(debug, 0))
# Create string from list.
plug_in_dir_paths = ':'.join(plug_in_packages_list)
temp = tempfile.NamedTemporaryFile()
temp_file_path = temp.name
temp2 = tempfile.NamedTemporaryFile()
temp_properties_file_path = temp2.name
if debug:
os.environ["PERF_TRACE"] = "1"
debug_string = " --quiet=0"
else:
debug_string = ""
loc_shell_rc = 0
sub_cmd_buf = "process_plug_in_packages.py" + debug_string +\
" --call_point=" + call_point + " --allow_shell_rc=" +\
str(shell_rc) + " --stop_on_plug_in_failure=" +\
str(stop_on_plug_in_failure) + " --stop_on_non_zero_rc=" +\
str(stop_on_non_zero_rc) + " " + plug_in_dir_paths
if quiet:
cmd_buf = sub_cmd_buf + " > " + temp_file_path + " 2>&1"
else:
cmd_buf = "set -o pipefail ; " + sub_cmd_buf + " 2>&1 | tee " +\
temp_file_path
if debug:
gp.print_issuing(cmd_buf)
else:
gp.print_timen("Processing " + call_point +
" call point programs.")
sub_proc = subprocess.Popen(cmd_buf, shell=True, executable='/bin/bash')
sub_proc.communicate()
proc_plug_pkg_rc = sub_proc.returncode
if return_history:
# Get the "Running" statements from the output.
regex = " Running [^/]+/cp_"
cmd_buf = "egrep '" + regex + "' " + temp_file_path
_, history = gc.shell_cmd(cmd_buf,
quiet=(not debug),
print_output=0,
show_err=0,
ignore_err=1)
history = [x + "\n" for x in filter(None, history.split("\n"))]
else:
history = []
# As process_plug_in_packages.py help text states, it will print the values of failed_plug_in_name and
# shell_rc in the following format:
# failed_plug_in_name: <failed plug-in value, if any>
# shell_rc: <shell return code value of last call point program>
# We want to obtain those values from the output. To make the task simpler, we'll start by grepping the
# output for lines that might fit such a format:
# A valid bash variable against the left margin followed by...
# - A colon followed by...
# - Zero or more spaces
bash_var_regex = "[_[:alpha:]][_[:alnum:]]*"
regex = "^" + bash_var_regex + ":[ ]*"
cmd_buf = "egrep '" + regex + "' " + temp_file_path + " > " +\
temp_properties_file_path
gp.dprint_issuing(cmd_buf)
grep_rc = os.system(cmd_buf)
# Next we call my_parm_file to create a properties dictionary.
properties = gm.my_parm_file(temp_properties_file_path)
# Finally, we access the 2 values that we need.
shell_rc = int(properties.get('shell_rc', '0x0000000000000000'), 16)
failed_plug_in_name = properties.get('failed_plug_in_name', '')
if proc_plug_pkg_rc != 0:
if quiet:
os.system("cat " + temp_file_path + " >&2")
if grep_rc != 0:
gp.print_var(grep_rc, gp.hexa())
gp.print_var(proc_plug_pkg_rc, gp.hexa())
gp.print_timen("Re-cap of plug-in failures:")
gc.cmd_fnc_u("egrep -A 1 '^failed_plug_in_name:[ ]+' " +
temp_properties_file_path + " | egrep -v '^\\--'",
quiet=1,
show_err=0)
rc = 1
if return_history:
return rc, shell_rc, failed_plug_in_name, history
else:
return rc, shell_rc, failed_plug_in_name
def wait_state(match_state=(),
wait_time="1 min",
interval="1 second",
invert=0,
openbmc_host="",
openbmc_username="",
openbmc_password="",
os_host="",
os_username="",
os_password="",
quiet=None):
r"""
Wait for the Open BMC machine's composite state to match the specified
state. On success, this keyword returns the machine's composite state as
a dictionary.
Description of argument(s):
match_state A dictionary whose key/value pairs are "state field"/
"state value". See check_state (above) for details.
This value may also be any string accepted by
return_state_constant (e.g. "standby_match_state").
In such a case this function will call
return_state_constant to convert it to a proper
dictionary as described above.
wait_time The total amount of time to wait for the desired state.
This value may be expressed in Robot Framework's time
format (e.g. 1 minute, 2 min 3 s, 4.5).
interval The amount of time between state checks.
This value may be expressed in Robot Framework's time
format (e.g. 1 minute, 2 min 3 s, 4.5).
invert If this flag is set, this function will for the state of
the machine to cease to match the match state.
openbmc_host The DNS name or IP address of the BMC.
This defaults to global ${OPENBMC_HOST}.
openbmc_username The username to be used to login to the BMC.
This defaults to global ${OPENBMC_USERNAME}.
openbmc_password The password to be used to login to the BMC.
This defaults to global ${OPENBMC_PASSWORD}.
os_host The DNS name or IP address of the operating system.
This defaults to global ${OS_HOST}.
os_username The username to be used to login to the OS.
This defaults to global ${OS_USERNAME}.
os_password The password to be used to login to the OS.
This defaults to global ${OS_PASSWORD}.
quiet Indicates whether status details should be written to the
console. Defaults to either global value of ${QUIET} or
to 1.
"""
quiet = int(gp.get_var_value(quiet, 0))
try:
match_state = return_state_constant(match_state)
except TypeError:
pass
if not quiet:
if invert:
alt_text = "cease to "
else:
alt_text = ""
gp.print_timen("Checking every " + str(interval) + " for up to "
+ str(wait_time) + " for the state of the machine to "
+ alt_text + "match the state shown below.")
gp.print_var(match_state)
if quiet:
print_string = ""
else:
print_string = "#"
debug = int(BuiltIn().get_variable_value("${debug}", "0"))
if debug:
# In debug we print state so no need to print the "#".
print_string = ""
check_state_quiet = 1 - debug
cmd_buf = ["Check State", match_state, "invert=${" + str(invert) + "}",
"print_string=" + print_string, "openbmc_host=" + openbmc_host,
"openbmc_username="******"openbmc_password="******"os_host=" + os_host,
"os_username="******"os_password="******"quiet=${" + str(check_state_quiet) + "}"]
gp.dprint_issuing(cmd_buf)
try:
state = BuiltIn().wait_until_keyword_succeeds(wait_time, interval,
*cmd_buf)
except AssertionError as my_assertion_error:
gp.printn()
message = my_assertion_error.args[0]
BuiltIn().fail(message)
if exit_wait_early_message:
# The global exit_wait_early_message was set by a signal handler
# indicating that we should fail.
message = exit_wait_early_message
# Clear the exit_wait_early_message variable for future use.
set_exit_wait_early_message("")
BuiltIn().fail(gp.sprint_error(message))
if not quiet:
gp.printn()
if invert:
gp.print_timen("The states no longer match:")
else:
gp.print_timen("The states match:")
gp.print_var(state)
return state
def run_key(keyword_buf, quiet=None, test_mode=None, ignore=0):
r"""
Run the given keyword, return the status and the keyword return values.
The advantage of using this function verses having robot simply run your
keyword is the handling of parameters like quiet, test_mode and ignore.
Description of arguments:
keyword_buf The keyword string to be run.
quiet Indicates whether this function should run
the pissuing function to print 'Issuing:
<keyword string>' to stdout.
test_mode If test_mode is set, this function will
not actually run the command. If quiet is
0, it will print a message indicating what
it would have run (e.g. "Issuing:
(test_mode) your command").
ignore Ignore errors from running keyword. If
this is 0, this function will fail with
whatever error occurred when running the
keyword.
Example usage from a robot script:
${status} ${ret_values}= Run Key My Keyword \ Arg1 \ Arg2
Note that to get robot to pass your command + args as a single string to
this function, you must escape extra spaces with a backslash.
Also note that ret_values is a python list:
ret_values:
ret_values[0]: value1
ret_values[1]: value2
"""
# Set these vars to default values if they are None.
quiet = int(gp.get_var_value(quiet, 0))
test_mode = int(gp.get_var_value(test_mode, 0))
ignore = int(ignore)
# Convert the keyword_buf into a list split wherever 2 or more spaces are
# found.
keyword_list = keyword_buf.split(' ')
# Strip spaces from each argument to make the output look clean and
# uniform.
keyword_list = [item.strip(' ') for item in keyword_list]
if not quiet:
# Join the list back into keyword_buf for the sake of output.
keyword_buf = ' '.join(keyword_list)
gp.pissuing(keyword_buf, test_mode)
if test_mode:
return 'PASS', ""
try:
status, ret_values = \
BuiltIn().run_keyword_and_ignore_error(*keyword_list)
except Exception as my_assertion_error:
status = "FAIL"
ret_values = my_assertion_error.args[0]
if not (status == 'PASS' or ignore):
# Output the error message to stderr.
BuiltIn().log_to_console(ret_values, stream='STDERR')
# Fail with the given error message.
BuiltIn().fail(ret_values)
return status, ret_values
