def wait_state(match_state=(),
wait_time="1 min",
interval="1 second",
invert=0,
openbmc_host="",
openbmc_username="",
openbmc_password="",
os_host="",
os_username="",
os_password="",
quiet=None):
r"""
Wait for the Open BMC machine's composite state to match the specified
state. On success, this keyword returns the machine's composite state as
a dictionary.
Description of argument(s):
match_state A dictionary whose key/value pairs are "state field"/
"state value". See check_state (above) for details.
This value may also be any string accepted by
return_state_constant (e.g. "standby_match_state").
In such a case this function will call
return_state_constant to convert it to a proper
dictionary as described above.
wait_time The total amount of time to wait for the desired state.
This value may be expressed in Robot Framework's time
format (e.g. 1 minute, 2 min 3 s, 4.5).
interval The amount of time between state checks.
This value may be expressed in Robot Framework's time
format (e.g. 1 minute, 2 min 3 s, 4.5).
invert If this flag is set, this function will for the state of
the machine to cease to match the match state.
openbmc_host The DNS name or IP address of the BMC.
This defaults to global ${OPENBMC_HOST}.
openbmc_username The username to be used to login to the BMC.
This defaults to global ${OPENBMC_USERNAME}.
openbmc_password The password to be used to login to the BMC.
This defaults to global ${OPENBMC_PASSWORD}.
os_host The DNS name or IP address of the operating system.
This defaults to global ${OS_HOST}.
os_username The username to be used to login to the OS.
This defaults to global ${OS_USERNAME}.
os_password The password to be used to login to the OS.
This defaults to global ${OS_PASSWORD}.
quiet Indicates whether status details should be written to the
console. Defaults to either global value of ${QUIET} or
to 1.
"""
quiet = int(gp.get_var_value(quiet, 0))
try:
match_state = return_state_constant(match_state)
except TypeError:
pass
if not quiet:
if invert:
alt_text = "cease to "
else:
alt_text = ""
gp.print_timen("Checking every " + str(interval) + " for up to "
+ str(wait_time) + " for the state of the machine to "
+ alt_text + "match the state shown below.")
gp.print_var(match_state)
if quiet:
print_string = ""
else:
print_string = "#"
debug = int(BuiltIn().get_variable_value("${debug}", "0"))
if debug:
# In debug we print state so no need to print the "#".
print_string = ""
check_state_quiet = 1 - debug
cmd_buf = ["Check State", match_state, "invert=${" + str(invert) + "}",
"print_string=" + print_string, "openbmc_host=" + openbmc_host,
"openbmc_username="******"openbmc_password="******"os_host=" + os_host,
"os_username="******"os_password="******"quiet=${" + str(check_state_quiet) + "}"]
gp.dprint_issuing(cmd_buf)
try:
state = BuiltIn().wait_until_keyword_succeeds(wait_time, interval,
*cmd_buf)
except AssertionError as my_assertion_error:
gp.printn()
message = my_assertion_error.args[0]
BuiltIn().fail(message)
if exit_wait_early_message:
# The global exit_wait_early_message was set by a signal handler
# indicating that we should fail.
message = exit_wait_early_message
# Clear the exit_wait_early_message variable for future use.
set_exit_wait_early_message("")
BuiltIn().fail(gp.sprint_error(message))
if not quiet:
gp.printn()
if invert:
gp.print_timen("The states no longer match:")
else:
gp.print_timen("The states match:")
gp.print_var(state)
return state
def get_state(openbmc_host="",
openbmc_username="",
openbmc_password="",
os_host="",
os_username="",
os_password="",
req_states=default_req_states,
quiet=None):
r"""
Get component states such as chassis state, bmc state, etc, put them into a
dictionary and return them to the caller.
Note that all substate values are strings.
Note: If elapsed_boot_time is included in req_states, it is the caller's
duty to call set_start_boot_seconds() in order to set global
start_boot_seconds. elapsed_boot_time is the current time minus
start_boot_seconds.
Description of argument(s):
openbmc_host The DNS name or IP address of the BMC.
This defaults to global ${OPENBMC_HOST}.
openbmc_username The username to be used to login to the BMC.
This defaults to global ${OPENBMC_USERNAME}.
openbmc_password The password to be used to login to the BMC.
This defaults to global ${OPENBMC_PASSWORD}.
os_host The DNS name or IP address of the operating system.
This defaults to global ${OS_HOST}.
os_username The username to be used to login to the OS.
This defaults to global ${OS_USERNAME}.
os_password The password to be used to login to the OS.
This defaults to global ${OS_PASSWORD}.
req_states This is a list of states whose values are being requested
by the caller.
quiet Indicates whether status details (e.g. curl commands)
should be written to the console.
Defaults to either global value of ${QUIET} or to 1.
"""
quiet = int(gp.get_var_value(quiet, 0))
# Set parm defaults where necessary and validate all parms.
if openbmc_host == "":
openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}")
error_message = gv.valid_value(openbmc_host, invalid_values=[None, ""])
if error_message != "":
BuiltIn().fail(gp.sprint_error(error_message))
if openbmc_username == "":
openbmc_username = BuiltIn().get_variable_value("${OPENBMC_USERNAME}")
error_message = gv.valid_value(openbmc_username, invalid_values=[None, ""])
if error_message != "":
BuiltIn().fail(gp.sprint_error(error_message))
if openbmc_password == "":
openbmc_password = BuiltIn().get_variable_value("${OPENBMC_PASSWORD}")
error_message = gv.valid_value(openbmc_password, invalid_values=[None, ""])
if error_message != "":
BuiltIn().fail(gp.sprint_error(error_message))
# NOTE: OS parms are optional.
if os_host == "":
os_host = BuiltIn().get_variable_value("${OS_HOST}")
if os_host is None:
os_host = ""
if os_username is "":
os_username = BuiltIn().get_variable_value("${OS_USERNAME}")
if os_username is None:
os_username = ""
if os_password is "":
os_password = BuiltIn().get_variable_value("${OS_PASSWORD}")
if os_password is None:
os_password = ""
invalid_req_states = [sub_state for sub_state in req_states
if sub_state not in valid_req_states]
if len(invalid_req_states) > 0:
error_message = "The following req_states are not supported:\n" +\
gp.sprint_var(invalid_req_states)
BuiltIn().fail(gp.sprint_error(error_message))
# Initialize all substate values supported by this function.
ping = 0
packet_loss = ''
uptime = ''
epoch_seconds = ''
elapsed_boot_time = ''
rest = ''
chassis = ''
requested_chassis = ''
bmc = ''
requested_bmc = ''
boot_progress = ''
operating_system = ''
host = ''
requested_host = ''
attempts_left = ''
# Get the component states.
if 'ping' in req_states:
# See if the OS pings.
rc, out_buf = gc.shell_cmd("ping -c 1 -w 2 " + openbmc_host,
print_output=0, show_err=0,
ignore_err=1)
if rc == 0:
ping = 1
if 'packet_loss' in req_states:
# See if the OS pings.
cmd_buf = "ping -c 5 -w 5 " + openbmc_host +\
" | egrep 'packet loss' | sed -re 's/.* ([0-9]+)%.*/\\1/g'"
rc, out_buf = gc.shell_cmd(cmd_buf,
print_output=0, show_err=0,
ignore_err=1)
if rc == 0:
packet_loss = out_buf.rstrip("\n")
if 'uptime' in req_states:
# Sometimes reading uptime results in a blank value. Call with
# wait_until_keyword_succeeds to ensure a non-blank value is obtained.
remote_cmd_buf = "read uptime filler 2>/dev/null < /proc/uptime" +\
" && [ ! -z \"${uptime}\" ] && echo ${uptime}"
cmd_buf = ["BMC Execute Command",
re.sub('\\$', '\\$', remote_cmd_buf), 'quiet=1',
'test_mode=0']
gp.qprint_issuing(cmd_buf, 0)
gp.qprint_issuing(remote_cmd_buf, 0)
try:
stdout, stderr, rc =\
BuiltIn().wait_until_keyword_succeeds("10 sec", "0 sec",
*cmd_buf)
if rc == 0 and stderr == "":
uptime = stdout
except AssertionError as my_assertion_error:
pass
if 'epoch_seconds' in req_states or 'elapsed_boot_time' in req_states:
date_cmd_buf = "date -u +%s"
if USE_BMC_EPOCH_TIME:
cmd_buf = ["BMC Execute Command", date_cmd_buf, 'quiet=${1}']
if not quiet:
gp.print_issuing(cmd_buf)
status, ret_values = \
BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
if status == "PASS":
stdout, stderr, rc = ret_values
if rc == 0 and stderr == "":
epoch_seconds = stdout.rstrip("\n")
else:
shell_rc, out_buf = gc.cmd_fnc_u(date_cmd_buf,
quiet=quiet,
print_output=0)
if shell_rc == 0:
epoch_seconds = out_buf.rstrip("\n")
if 'elapsed_boot_time' in req_states:
global start_boot_seconds
elapsed_boot_time = int(epoch_seconds) - start_boot_seconds
master_req_rest = ['rest', 'host', 'requested_host', 'operating_system',
'attempts_left', 'boot_progress', 'chassis',
'requested_chassis' 'bmc' 'requested_bmc']
req_rest = [sub_state for sub_state in req_states if sub_state in
master_req_rest]
need_rest = (len(req_rest) > 0)
state = DotDict()
if need_rest:
cmd_buf = ["Read Properties", SYSTEM_STATE_URI + "enumerate",
"quiet=${" + str(quiet) + "}"]
gp.dprint_issuing(cmd_buf)
status, ret_values = \
BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
if status == "PASS":
state['rest'] = '1'
else:
state['rest'] = '0'
if int(state['rest']):
for url_path in ret_values:
for attr_name in ret_values[url_path]:
# Create a state key value based on the attr_name.
try:
ret_values[url_path][attr_name] = \
re.sub(r'.*\.', "",
ret_values[url_path][attr_name])
except TypeError:
pass
# Do some key name manipulations.
new_attr_name = re.sub(r'^Current|(State|Transition)$',
"", attr_name)
new_attr_name = re.sub(r'BMC', r'Bmc', new_attr_name)
new_attr_name = re.sub(r'([A-Z][a-z])', r'_\1',
new_attr_name)
new_attr_name = new_attr_name.lower().lstrip("_")
new_attr_name = re.sub(r'power', r'chassis', new_attr_name)
if new_attr_name in req_states:
state[new_attr_name] = ret_values[url_path][attr_name]
for sub_state in req_states:
if sub_state in state:
continue
if sub_state.startswith("os_"):
# We pass "os_" requests on to get_os_state.
continue
cmd_buf = "state['" + sub_state + "'] = str(" + sub_state + ")"
exec(cmd_buf)
if os_host == "":
# The caller has not specified an os_host so as far as we're concerned,
# it doesn't exist.
return state
os_req_states = [sub_state for sub_state in req_states
if sub_state.startswith('os_')]
if len(os_req_states) > 0:
# The caller has specified an os_host and they have requested
# information on os substates.
# Based on the information gathered on bmc, we'll try to make a
# determination of whether the os is even up. We'll pass the result
# of that assessment to get_os_state to enhance performance.
os_up_match = DotDict()
for sub_state in master_os_up_match:
if sub_state in req_states:
os_up_match[sub_state] = master_os_up_match[sub_state]
os_up = compare_states(state, os_up_match)
os_state = get_os_state(os_host=os_host,
os_username=os_username,
os_password=os_password,
req_states=os_req_states,
os_up=os_up,
quiet=quiet)
# Append os_state dictionary to ours.
state.update(os_state)
return state
def rprocess_plug_in_packages(plug_in_packages_list=None,
call_point="setup",
shell_rc="0x00000000",
stop_on_plug_in_failure=1,
stop_on_non_zero_rc=0,
release_type="obmc",
quiet=None,
debug=None,
return_history=False):
r"""
Call the external process_plug_in_packages.py to process the plug-in packages. Return the following:
rc The return code - 0 = PASS, 1 = FAIL.
shell_rc The shell return code returned by process_plug_in_packages.py.
failed_plug_in_name The failed plug in name (if any).
Description of arguments:
plug_in_packages_list A python list of plug-in directory paths.
call_point The call point program to be called for each plug-in package (e.g.
post_boot). This name should not include the "cp_" prefix.
shell_rc The user may supply a value other than zero to indicate an acceptable
non-zero return code. For example, if this value equals 0x00000200, it
means that for each plug-in call point that runs, a 0x00000200 will not
be counted as a failure.
stop_on_plug_in_failure If this parameter is set to 1, this program will stop and return non-zero
if the call point program from any plug-in directory fails. Conversely,
if it is set to false, this program will run the call point program from
each and every plug-in directory regardless of their return values.
Typical example cases where you'd want to run all plug-in call points
regardless of success or failure would be "cleanup" or "ffdc" call points.
stop_on_non_zero_rc If this parm is set to 1 and a plug-in call point program returns a valid
non-zero return code (see "shell_rc" parm above), this program will stop
processing and return 0 (success). Since this constitutes a successful
exit, this would normally be used where the caller wishes to stop
processing if one of the plug-in directory call point programs returns a
special value indicating that some special case has been found. An
example might be in calling some kind of "check_errl" call point program.
Such a call point program might return a 2 (i.e. 0x00000200) to indicate
that a given error log entry was found in an "ignore" list and is
therefore to be ignored. That being the case, no other "check_errl" call
point program would need to be called.
release_type The type of release being tested (e.g. "obmc", "op", "fips"). This
influences which integrated plug-ins are selected.
quiet If quiet is set to 1, this function will NOT write status messages to
stdout. This will default to the global quiet program parm or to 0.
debug If this parameter is set to 1, this function will print additional debug
information. This is mainly to be used by the developer of this
function. This will default to the global quiet program parm or to 0.
return_history In addition to rc, shell_rc and failed_plug_in_name, return a list
containing historical output that looks like the following:
history:
history[0]: #(CDT) 2018/10/30 12:25:49 - Running OBMC_Sample/cp_post_stack
"""
rc = 0
plug_in_packages_list = gp.get_var_value(plug_in_packages_list, [])
# If there are no plug-in packages to process, return successfully.
if len(plug_in_packages_list) == 0:
if return_history:
return 0, 0, "", []
else:
return 0, 0, ""
quiet = int(gp.get_var_value(quiet, 0))
debug = int(gp.get_var_value(debug, 0))
# Create string from list.
plug_in_dir_paths = ':'.join(plug_in_packages_list)
temp = tempfile.NamedTemporaryFile()
temp_file_path = temp.name
temp2 = tempfile.NamedTemporaryFile()
temp_properties_file_path = temp2.name
if debug:
os.environ["PERF_TRACE"] = "1"
debug_string = " --quiet=0"
else:
debug_string = ""
loc_shell_rc = 0
sub_cmd_buf = "process_plug_in_packages.py" + debug_string +\
" --call_point=" + call_point + " --allow_shell_rc=" +\
str(shell_rc) + " --stop_on_plug_in_failure=" +\
str(stop_on_plug_in_failure) + " --stop_on_non_zero_rc=" +\
str(stop_on_non_zero_rc) + " " + plug_in_dir_paths
if quiet:
cmd_buf = sub_cmd_buf + " > " + temp_file_path + " 2>&1"
else:
cmd_buf = "set -o pipefail ; " + sub_cmd_buf + " 2>&1 | tee " +\
temp_file_path
if debug:
gp.print_issuing(cmd_buf)
else:
gp.print_timen("Processing " + call_point +
" call point programs.")
sub_proc = subprocess.Popen(cmd_buf, shell=True, executable='/bin/bash')
sub_proc.communicate()
proc_plug_pkg_rc = sub_proc.returncode
if return_history:
# Get the "Running" statements from the output.
regex = " Running [^/]+/cp_"
cmd_buf = "egrep '" + regex + "' " + temp_file_path
_, history = gc.shell_cmd(cmd_buf,
quiet=(not debug),
print_output=0,
show_err=0,
ignore_err=1)
history = [x + "\n" for x in filter(None, history.split("\n"))]
else:
history = []
# As process_plug_in_packages.py help text states, it will print the values of failed_plug_in_name and
# shell_rc in the following format:
# failed_plug_in_name: <failed plug-in value, if any>
# shell_rc: <shell return code value of last call point program>
# We want to obtain those values from the output. To make the task simpler, we'll start by grepping the
# output for lines that might fit such a format:
# A valid bash variable against the left margin followed by...
# - A colon followed by...
# - Zero or more spaces
bash_var_regex = "[_[:alpha:]][_[:alnum:]]*"
regex = "^" + bash_var_regex + ":[ ]*"
cmd_buf = "egrep '" + regex + "' " + temp_file_path + " > " +\
temp_properties_file_path
gp.dprint_issuing(cmd_buf)
grep_rc = os.system(cmd_buf)
# Next we call my_parm_file to create a properties dictionary.
properties = gm.my_parm_file(temp_properties_file_path)
# Finally, we access the 2 values that we need.
shell_rc = int(properties.get('shell_rc', '0x0000000000000000'), 16)
failed_plug_in_name = properties.get('failed_plug_in_name', '')
if proc_plug_pkg_rc != 0:
if quiet:
os.system("cat " + temp_file_path + " >&2")
if grep_rc != 0:
gp.print_var(grep_rc, gp.hexa())
gp.print_var(proc_plug_pkg_rc, gp.hexa())
gp.print_timen("Re-cap of plug-in failures:")
gc.cmd_fnc_u("egrep -A 1 '^failed_plug_in_name:[ ]+' " +
temp_properties_file_path + " | egrep -v '^\\--'",
quiet=1,
show_err=0)
rc = 1
if return_history:
return rc, shell_rc, failed_plug_in_name, history
else:
return rc, shell_rc, failed_plug_in_name
def set_ffdc_defaults(ffdc_dir_path=None, ffdc_prefix=None):
r"""
Set a default value for ffdc_dir_path and ffdc_prefix if they don't
already have values. Return both values.
Description of arguments:
ffdc_dir_path The dir path where FFDC data should be put.
ffdc_prefix The prefix to be given to each FFDC file name generated.
NOTE: If global variable ffdc_dir_path_style is set to ${1}, this function
will create default values in a newer way. Otherwise, its behavior
will remain unchanged.
"""
# Note: Several subordinate functions like 'Get Test Dir and Name' and
# 'Header Message' expect global variable FFDC_TIME to be set.
cmd_buf = ["Get Current Time Stamp"]
gp.dprint_issuing(cmd_buf)
FFDC_TIME = BuiltIn().run_keyword(*cmd_buf)
BuiltIn().set_global_variable("${FFDC_TIME}", FFDC_TIME)
ffdc_dir_path_style = BuiltIn().get_variable_value(
"${ffdc_dir_path_style}")
if ffdc_dir_path is None:
if ffdc_dir_path_style:
try:
ffdc_dir_path = os.environ['FFDC_DIR_PATH']
except KeyError:
ffdc_dir_path = os.path.dirname(
BuiltIn().get_variable_value("${LOG_FILE}")) + "/"
else:
FFDC_LOG_PATH = os.getcwd() + "/logs/"
if FFDC_LOG_PATH is None:
FFDC_LOG_PATH = ""
if FFDC_LOG_PATH == "":
FFDC_LOG_PATH = os.path.dirname(
BuiltIn().get_variable_value("${LOG_FILE}")) + "/"
error_message = gv.valid_value(FFDC_LOG_PATH,
var_name="FFDC_LOG_PATH")
if error_message != "":
error_message = gp.sprint_error_report(error_message)
BuiltIn().fail(error_message)
FFDC_LOG_PATH = os.path.normpath(FFDC_LOG_PATH) + os.sep
cmd_buf = ["Get Test Dir and Name"]
gp.print_issuing(cmd_buf)
suitename, testname = BuiltIn().run_keyword(*cmd_buf)
ffdc_dir_path = FFDC_LOG_PATH + suitename + "/" + testname + "/"
# Add trailing slash.
ffdc_dir_path = os.path.normpath(ffdc_dir_path) + os.sep
if ffdc_prefix is None:
FFDC_TIME = BuiltIn().get_variable_value("${FFDC_TIME}")
if ffdc_prefix is None:
if ffdc_dir_path_style:
OPENBMC_HOST = BuiltIn().get_variable_value("${OPENBMC_HOST}")
OPENBMC_NICKNAME = BuiltIn().get_variable_value(
"${OPENBMC_NICKNAME}", default=OPENBMC_HOST)
ffdc_prefix = OPENBMC_NICKNAME + "." + FFDC_TIME[2:8] + "." +\
FFDC_TIME[8:14] + "."
else:
ffdc_prefix = FFDC_TIME + "_"
BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
BuiltIn().set_global_variable("${FFDC_PREFIX}", ffdc_prefix)
return ffdc_dir_path, ffdc_prefix
def my_run_keywords(lib_file_path, keyword_string, quiet=0, test_mode=0):
r"""
Run the keywords in the keyword string.
Description of arguments:
lib_file_path The path to a library or resource needed to run the
keywords. This may contain a colon-delimited list of
library/resource paths.
keyword_string The keyword string to be run by this function. If this
keyword string contains " ; " anywhere, it will be taken to
be multiple keyword strings. Each keyword may also include
a variable assignment. Example:
${my_var}= My Keyword
quiet If this parameter is set to "1", this program will print
only essential information, i.e. it will not echo
parameters, echo commands, print the total run time, etc.
test_mode This means that this program should go through all the
motions but not actually do anything substantial.
"""
# NOTE: During code review the following question was raised: Why support
# 1) variable assignments 2) multiple keywords? Couldn't a user simply
# call this program twice to get what they need. If necessary, the user
# could take the output of the first call and specify it as a literal on
# the second call.
#
# However, this approach would not work in all cases. The following case
# would be such an example:
# Let's say the first keyword string is as follows:
# Create Dictionary foo=bar
# You wish to take the output of that call and specify it as a literal
# value when running the following:
# Want Dictionary parm=<literal dictionary specification>
# The problem is that there is no way to specify a dictionary as a
# literal in Robot Framework.
# By having this program support variable assignments and multiple
# keywords, the user can invoke it with the following keyword string.
# ${my_dict}= Create Dictionary foo=bar ; Want Dictionary ${my_dict}
# The user can pass multiple lib/resource paths by separating them with a
# colon.
lib_file_path_list = lib_file_path.split(":")
# Get rid of empty entry if it exists.
if lib_file_path_list[0] == "":
del lib_file_path_list[0]
for lib_file_path in lib_file_path_list:
if lib_file_path.endswith(".py"):
gp.dprint_issuing("import_library(\"" + lib_file_path + "\")")
BuiltIn().import_library(lib_file_path)
else:
gp.dprint_issuing("my_import_resource(\"" + lib_file_path + "\")")
gru.my_import_resource(lib_file_path)
# The user can pass multiple keyword strings by separating them with " ; ".
keyword_list = keyword_string.split(" ; ")
for keyword_string in keyword_list:
cmd_buf = keyword_string.split(" ")
if re.match(r"\$\{", cmd_buf[0]):
# This looks like an assignment (e.g. ${var}= <keyword>).
# We'll extract the variable name, remove element 0 from
# cmd_buf and set the global variable with the results
# after running the keyword.
var_name = cmd_buf[0].strip("${}=")
del cmd_buf[0]
else:
var_name = ""
if not quiet:
gp.print_issuing(cmd_buf, test_mode)
if test_mode:
continue
output = BuiltIn().run_keyword(*cmd_buf)
if var_name != "":
BuiltIn().set_global_variable("${" + var_name + "}", output)
else:
if output is not None:
gp.gp_print(output)
