def print_error_logs(error_logs, key_list=None):
r"""
Print the error logs to the console screen.
This function provides the following benefits:
- It will specify print_var parms for the caller (e.g. hex=1).
- It is much easier to call this function than to generate the desired code
directly from a robot script.
Description of argument(s):
error_logs An error log dictionary such as the one
returned by the 'Get Error Logs' keyword.
key_list The list of keys to be printed. This may
be specified as either a python list
or a space-delimited string. In the
latter case, this function will convert
it to a python list. See the sprint_varx
function prolog for additionatl details.
Example use from a python script:
${error_logs}= Get Error Logs
Print Error Logs ${error_logs} Message Timestamp
Sample output:
error_logs:
[/xyz/openbmc_project/logging/entry/3]:
[Timestamp]: 1521738335735
[Message]:
xyz.openbmc_project.Inventory.Error.Nonfunctional
[/xyz/openbmc_project/logging/entry/2]:
[Timestamp]: 1521738334637
[Message]:
xyz.openbmc_project.Inventory.Error.Nonfunctional
[/xyz/openbmc_project/logging/entry/1]:
[Timestamp]: 1521738300696
[Message]:
xyz.openbmc_project.Inventory.Error.Nonfunctional
[/xyz/openbmc_project/logging/entry/4]:
[Timestamp]: 1521738337915
[Message]:
xyz.openbmc_project.Inventory.Error.Nonfunctional
Another example call using a robot list:
${error_logs}= Get Error Logs
${key_list}= Create List Message Timestamp Severity
Print Error Logs ${error_logs} ${key_list}
"""
if key_list is not None:
if type(key_list) in (str, unicode):
key_list = key_list.split(" ")
key_list.insert(0, var.BMC_LOGGING_ENTRY + ".*")
gp.print_var(error_logs, hex=1, key_list=key_list)
def print_error_logs(error_logs, key_list=None):
r"""
Print the error logs to the console screen.
This function provides the following benefits:
- It will specify print_var parms for the caller (e.g. hex=1).
- It is much easier to call this function than to generate the desired code
directly from a robot script.
Description of argument(s):
error_logs An error log dictionary such as the one
returned by the 'Get Error Logs' keyword.
key_list The list of keys to be printed. This may
be specified as either a python list
or a space-delimited string. In the
latter case, this function will convert
it to a python list. See the sprint_varx
function prolog for additionatl details.
Example use from a python script:
${error_logs}= Get Error Logs
Print Error Logs ${error_logs} Message Timestamp
Sample output:
error_logs:
[/xyz/openbmc_project/logging/entry/3]:
[Timestamp]: 1521738335735
[Message]:
xyz.openbmc_project.Inventory.Error.Nonfunctional
[/xyz/openbmc_project/logging/entry/2]:
[Timestamp]: 1521738334637
[Message]:
xyz.openbmc_project.Inventory.Error.Nonfunctional
[/xyz/openbmc_project/logging/entry/1]:
[Timestamp]: 1521738300696
[Message]:
xyz.openbmc_project.Inventory.Error.Nonfunctional
[/xyz/openbmc_project/logging/entry/4]:
[Timestamp]: 1521738337915
[Message]:
xyz.openbmc_project.Inventory.Error.Nonfunctional
Another example call using a robot list:
${error_logs}= Get Error Logs
${key_list}= Create List Message Timestamp Severity
Print Error Logs ${error_logs} ${key_list}
"""
if key_list is not None:
if type(key_list) in (str, unicode):
key_list = key_list.split(" ")
key_list.insert(0, var.BMC_LOGGING_ENTRY + ".*")
gp.print_var(error_logs, hex=1, key_list=key_list)
def verify_image_upload(image_version,
timeout=3):
r"""
Verify the image was uploaded correctly and that it created
a valid d-bus object. If the first check for the image
fails, try again until we reach the timeout.
Description of argument(s):
image_version The version from the image's manifest file
(e.g. "IBM-witherspoon-redbud-ibm-OP9_v1.17_1.68").
timeout How long, in minutes, to keep trying to find the
image on the BMC. Default is 3 minutes.
"""
image_path = get_image_path(image_version)
image_version_id = image_path.split("/")[-2]
keyword.run_key_u("Open Connection And Log In")
image_purpose = get_image_purpose(image_path + "MANIFEST")
if (image_purpose == var.VERSION_PURPOSE_BMC or
image_purpose == var.VERSION_PURPOSE_HOST):
uri = var.SOFTWARE_VERSION_URI + image_version_id
ret_values = ""
for itr in range(timeout * 2):
status, ret_values = \
keyword.run_key("Read Attribute " + uri + " Activation")
if ((ret_values == var.READY) or (ret_values == var.INVALID)
or (ret_values == var.ACTIVE)):
return True, image_version_id
else:
time.sleep(30)
# If we exit the for loop, the timeout has been reached
gp.print_var(ret_values)
return False, None
else:
gp.print_var(image_purpose)
return False, None
def verify_image_upload(image_version,
timeout=3):
r"""
Verify the image was uploaded correctly and that it created
a valid d-bus object. If the first check for the image
fails, try again until we reach the timeout.
Description of argument(s):
image_version The version from the image's manifest file
(e.g. "v2.2-253-g00050f1").
timeout How long, in minutes, to keep trying to
find the image on the BMC. Default is 3 minutes.
"""
image_path = get_image_path(image_version)
image_version_id = image_path.split("/")[-2]
keyword.run_key_u("Open Connection And Log In")
image_purpose = get_image_purpose(image_path + "MANIFEST")
if (image_purpose == var.VERSION_PURPOSE_BMC
or image_purpose == var.VERSION_PURPOSE_HOST):
uri = var.SOFTWARE_VERSION_URI + image_version_id
ret_values = ""
for itr in range(timeout * 2):
status, ret_values = \
keyword.run_key("Read Attribute " + uri + " Activation")
if ((ret_values == var.READY) or (ret_values == var.INVALID)
or (ret_values == var.ACTIVE)):
return True, image_version_id
else:
time.sleep(30)
# If we exit the for loop, the timeout has been reached
gp.print_var(ret_values)
return False, None
else:
gp.print_var(image_purpose)
return False, None
def wait_state(match_state=(),
wait_time="1 min",
interval="1 second",
invert=0,
openbmc_host="",
openbmc_username="",
openbmc_password="",
os_host="",
os_username="",
os_password="",
quiet=None):
r"""
Wait for the Open BMC machine's composite state to match the specified
state. On success, this keyword returns the machine's composite state as
a dictionary.
Description of argument(s):
match_state A dictionary whose key/value pairs are "state field"/
"state value". See check_state (above) for details.
This value may also be any string accepted by
return_state_constant (e.g. "standby_match_state").
In such a case this function will call
return_state_constant to convert it to a proper
dictionary as described above.
wait_time The total amount of time to wait for the desired state.
This value may be expressed in Robot Framework's time
format (e.g. 1 minute, 2 min 3 s, 4.5).
interval The amount of time between state checks.
This value may be expressed in Robot Framework's time
format (e.g. 1 minute, 2 min 3 s, 4.5).
invert If this flag is set, this function will for the state of
the machine to cease to match the match state.
openbmc_host The DNS name or IP address of the BMC.
This defaults to global ${OPENBMC_HOST}.
openbmc_username The username to be used to login to the BMC.
This defaults to global ${OPENBMC_USERNAME}.
openbmc_password The password to be used to login to the BMC.
This defaults to global ${OPENBMC_PASSWORD}.
os_host The DNS name or IP address of the operating system.
This defaults to global ${OS_HOST}.
os_username The username to be used to login to the OS.
This defaults to global ${OS_USERNAME}.
os_password The password to be used to login to the OS.
This defaults to global ${OS_PASSWORD}.
quiet Indicates whether status details should be written to the
console. Defaults to either global value of ${QUIET} or
to 1.
"""
quiet = int(gp.get_var_value(quiet, 0))
try:
match_state = return_state_constant(match_state)
except TypeError:
pass
if not quiet:
if invert:
alt_text = "cease to "
else:
alt_text = ""
gp.print_timen("Checking every " + str(interval) + " for up to "
+ str(wait_time) + " for the state of the machine to "
+ alt_text + "match the state shown below.")
gp.print_var(match_state)
if quiet:
print_string = ""
else:
print_string = "#"
debug = int(BuiltIn().get_variable_value("${debug}", "0"))
if debug:
# In debug we print state so no need to print the "#".
print_string = ""
check_state_quiet = 1 - debug
cmd_buf = ["Check State", match_state, "invert=${" + str(invert) + "}",
"print_string=" + print_string, "openbmc_host=" + openbmc_host,
"openbmc_username="******"openbmc_password="******"os_host=" + os_host,
"os_username="******"os_password="******"quiet=${" + str(check_state_quiet) + "}"]
gp.dprint_issuing(cmd_buf)
try:
state = BuiltIn().wait_until_keyword_succeeds(wait_time, interval,
*cmd_buf)
except AssertionError as my_assertion_error:
gp.printn()
message = my_assertion_error.args[0]
BuiltIn().fail(message)
if exit_wait_early_message:
# The global exit_wait_early_message was set by a signal handler
# indicating that we should fail.
message = exit_wait_early_message
# Clear the exit_wait_early_message variable for future use.
set_exit_wait_early_message("")
BuiltIn().fail(gp.sprint_error(message))
if not quiet:
gp.printn()
if invert:
gp.print_timen("The states no longer match:")
else:
gp.print_timen("The states match:")
gp.print_var(state)
return state
def check_state(match_state,
invert=0,
print_string="",
openbmc_host="",
openbmc_username="",
openbmc_password="",
os_host="",
os_username="",
os_password="",
quiet=None):
r"""
Check that the Open BMC machine's composite state matches the specified
state. On success, this keyword returns the machine's composite state as a
dictionary.
Description of argument(s):
match_state A dictionary whose key/value pairs are "state field"/
"state value". The state value is interpreted as a
regular expression. Example call from robot:
${match_state}= Create Dictionary chassis=^On$
... bmc=^Ready$
... boot_progress=^OSStart$
${state}= Check State &{match_state}
invert If this flag is set, this function will succeed if the
states do NOT match.
print_string This function will print this string to the console prior
to getting the state.
openbmc_host The DNS name or IP address of the BMC.
This defaults to global ${OPENBMC_HOST}.
openbmc_username The username to be used to login to the BMC.
This defaults to global ${OPENBMC_USERNAME}.
openbmc_password The password to be used to login to the BMC.
This defaults to global ${OPENBMC_PASSWORD}.
os_host The DNS name or IP address of the operating system.
This defaults to global ${OS_HOST}.
os_username The username to be used to login to the OS.
This defaults to global ${OS_USERNAME}.
os_password The password to be used to login to the OS.
This defaults to global ${OS_PASSWORD}.
quiet Indicates whether status details should be written to the
console. Defaults to either global value of ${QUIET} or
to 1.
"""
quiet = int(gp.get_var_value(quiet, 0))
gp.gp_print(print_string)
try:
match_state = return_state_constant(match_state)
except TypeError:
pass
req_states = list(match_state.keys())
# Remove special-case match key from req_states.
if expressions_key() in req_states:
req_states.remove(expressions_key())
# Initialize state.
state = get_state(openbmc_host=openbmc_host,
openbmc_username=openbmc_username,
openbmc_password=openbmc_password,
os_host=os_host,
os_username=os_username,
os_password=os_password,
req_states=req_states,
quiet=quiet)
if not quiet:
gp.print_var(state)
if exit_wait_early_message != "":
# The exit_wait_early_message has been set by a signal handler so we
# will exit "successfully". It is incumbent upon the calling function
# (e.g. wait_state) to check/clear this variable and to fail
# appropriately.
return state
match = compare_states(state, match_state)
if invert and match:
fail_msg = "The current state of the machine matches the match" +\
" state:\n" + gp.sprint_varx("state", state)
BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
elif not invert and not match:
fail_msg = "The current state of the machine does NOT match the" +\
" match state:\n" +\
gp.sprint_varx("state", state)
BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
return state
def check_state(match_state,
invert=0,
print_string="",
openbmc_host="",
openbmc_username="",
openbmc_password="",
os_host="",
os_username="",
os_password="",
quiet=None):
r"""
Check that the Open BMC machine's composite state matches the specified
state. On success, this keyword returns the machine's composite state as a
dictionary.
Description of arguments:
match_state A dictionary whose key/value pairs are "state field"/
"state value". The state value is interpreted as a
regular expression. Example call from robot:
${match_state}= Create Dictionary chassis=^On$
... bmc=^Ready$
... boot_progress=^OSStart$
${state}= Check State &{match_state}
invert If this flag is set, this function will succeed if the
states do NOT match.
print_string This function will print this string to the console prior
to getting the state.
openbmc_host The DNS name or IP address of the BMC.
This defaults to global ${OPENBMC_HOST}.
openbmc_username The username to be used to login to the BMC.
This defaults to global ${OPENBMC_USERNAME}.
openbmc_password The password to be used to login to the BMC.
This defaults to global ${OPENBMC_PASSWORD}.
os_host The DNS name or IP address of the operating system.
This defaults to global ${OS_HOST}.
os_username The username to be used to login to the OS.
This defaults to global ${OS_USERNAME}.
os_password The password to be used to login to the OS.
This defaults to global ${OS_PASSWORD}.
quiet Indicates whether status details should be written to the
console. Defaults to either global value of ${QUIET} or
to 1.
"""
quiet = int(gp.get_var_value(quiet, 0))
grp.rprint(print_string)
req_states = match_state.keys()
# Initialize state.
state = get_state(openbmc_host=openbmc_host,
openbmc_username=openbmc_username,
openbmc_password=openbmc_password,
os_host=os_host,
os_username=os_username,
os_password=os_password,
req_states=req_states,
quiet=quiet)
if not quiet:
gp.print_var(state)
match = compare_states(state, match_state)
if invert and match:
fail_msg = "The current state of the machine matches the match" +\
" state:\n" + gp.sprint_varx("state", state)
BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
elif not invert and not match:
fail_msg = "The current state of the machine does NOT match the" +\
" match state:\n" +\
gp.sprint_varx("state", state)
BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
return state
def select_boot():
r"""
Select a boot test to be run based on our current state and return the
chosen boot type.
Description of arguments:
state The state of the machine.
"""
global boot_stack
gp.qprint_timen("Selecting a boot test.")
my_get_state()
stack_popped = 0
if len(boot_stack) > 0:
stack_popped = 1
gp.qprint_dashes()
gp.qprint_var(boot_stack)
gp.qprint_dashes()
skip_boot_printed = 0
while len(boot_stack) > 0:
boot_candidate = boot_stack.pop()
if stack_mode == 'normal':
break
else:
if st.compare_states(state, boot_table[boot_candidate]['end']):
if not skip_boot_printed:
gp.print_var(stack_mode)
gp.printn()
gp.print_timen("Skipping the following boot tests" +
" which are unnecessary since their" +
" required end states match the" +
" current machine state:")
skip_boot_printed = 1
gp.print_var(boot_candidate)
boot_candidate = ""
if boot_candidate == "":
gp.qprint_dashes()
gp.qprint_var(boot_stack)
gp.qprint_dashes()
return boot_candidate
if st.compare_states(state, boot_table[boot_candidate]['start']):
gp.qprint_timen("The machine state is valid for a '" +
boot_candidate + "' boot test.")
gp.qprint_dashes()
gp.qprint_var(boot_stack)
gp.qprint_dashes()
return boot_candidate
else:
gp.qprint_timen("The machine state does not match the required" +
" starting state for a '" + boot_candidate +
"' boot test:")
gp.print_varx("boot_table[" + boot_candidate + "][start]",
boot_table[boot_candidate]['start'], 1)
boot_stack.append(boot_candidate)
popped_boot = boot_candidate
# Loop through your list selecting a boot_candidates
boot_candidates = []
for boot_candidate in boot_list:
if st.compare_states(state, boot_table[boot_candidate]['start']):
if stack_popped:
if st.compare_states(boot_table[boot_candidate]['end'],
boot_table[popped_boot]['start']):
boot_candidates.append(boot_candidate)
else:
boot_candidates.append(boot_candidate)
if len(boot_candidates) == 0:
gp.qprint_timen("The user's boot list contained no boot tests" +
" which are valid for the current machine state.")
boot_candidate = default_power_on
if not st.compare_states(state, boot_table[default_power_on]['start']):
boot_candidate = default_power_off
boot_candidates.append(boot_candidate)
gp.qprint_timen("Using default '" + boot_candidate +
"' boot type to transition to valid state.")
gp.dprint_var(boot_candidates)
# Randomly select a boot from the candidate list.
boot = random.choice(boot_candidates)
return boot
def execute_ssh_command(cmd_buf,
open_connection_args={},
login_args={},
print_out=0,
print_err=0,
ignore_err=1,
fork=0,
quiet=None,
test_mode=None):
r"""
Run the given command in an SSH session and return the stdout, stderr and
the return code.
If there is no open SSH connection, this function will connect and login.
Likewise, if the caller has not yet logged in to the connection, this
function will do the login.
NOTE: There is special handling when open_connection_args['alias'] equals
"device_connection".
- A write, rather than an execute_command, is done.
- Only stdout is returned (no stderr or rc).
- print_err, ignore_err and fork are not supported.
Description of arguments:
cmd_buf The command string to be run in an SSH
session.
open_connection_args A dictionary of arg names and values which
are legal to pass to the SSHLibrary
open_connection function as parms/args.
At a minimum, this should contain a 'host'
entry.
login_args A dictionary containing the key/value
pairs which are acceptable to the
SSHLibrary login function as parms/args.
At a minimum, this should contain a
'username' and a 'password' entry.
print_out If this is set, this function will print
the stdout/stderr generated by the shell
command.
print_err If show_err is set, this function will
print a standardized error report if the
shell command returns non-zero.
ignore_err Indicates that errors encountered on the
sshlib.execute_command are to be ignored.
fork Indicates that sshlib.start is to be used
rather than sshlib.execute_command.
quiet Indicates whether this function should run
the pissuing() function which prints an
"Issuing: <cmd string>" to stdout. This
defaults to the global quiet value.
test_mode If test_mode is set, this function will
not actually run the command. This
defaults to the global test_mode value.
"""
gp.lprint_executing()
# Obtain default values.
quiet = int(gp.get_var_value(quiet, 0))
test_mode = int(gp.get_var_value(test_mode, 0))
if not quiet:
gp.pissuing(cmd_buf, test_mode)
gp.lpissuing(cmd_buf, test_mode)
if test_mode:
return "", "", 0
global sshlib
max_exec_cmd_attempts = 2
# Look for existing SSH connection.
# Prepare a search connection dictionary.
search_connection_args = open_connection_args.copy()
# Remove keys that don't work well for searches.
search_connection_args.pop("timeout", None)
connection = find_connection(search_connection_args)
if connection:
gp.lprint_timen("Found the following existing connection:")
gp.lprintn(sprint_connection(connection))
if connection.alias == "":
index_or_alias = connection.index
else:
index_or_alias = connection.alias
gp.lprint_timen("Switching to existing connection: \""
+ str(index_or_alias) + "\".")
sshlib.switch_connection(index_or_alias)
else:
gp.lprint_timen("Connecting to " + open_connection_args['host'] + ".")
cix = sshlib.open_connection(**open_connection_args)
try:
login_ssh(login_args)
except Exception as login_exception:
except_type, except_value, except_traceback = sys.exc_info()
rc = 1
stderr = str(except_value)
stdout = ""
max_exec_cmd_attempts = 0
for exec_cmd_attempt_num in range(1, max_exec_cmd_attempts + 1):
gp.lprint_var(exec_cmd_attempt_num)
try:
if fork:
sshlib.start_command(cmd_buf)
else:
if open_connection_args['alias'] == "device_connection":
stdout = sshlib.write(cmd_buf)
stderr = ""
rc = 0
else:
stdout, stderr, rc = \
sshlib.execute_command(cmd_buf,
return_stdout=True,
return_stderr=True,
return_rc=True)
except Exception as execute_exception:
except_type, except_value, except_traceback = sys.exc_info()
gp.lprint_var(except_type)
gp.lprint_varx("except_value", str(except_value))
if except_type is exceptions.AssertionError and\
re.match(r"Connection not open", str(except_value)):
login_ssh(login_args)
# Now we must continue to next loop iteration to retry the
# execute_command.
continue
if (except_type is paramiko.ssh_exception.SSHException
and re.match(r"SSH session not active", str(except_value))) or\
(except_type is socket.error
and re.match(r"\[Errno 104\] Connection reset by peer",
str(except_value))):
# Close and re-open a connection.
# Note: close_connection() doesn't appear to get rid of the
# connection. It merely closes it. Since there is a concern
# about over-consumption of resources, we use
# close_all_connections() which also gets rid of all
# connections.
gp.lprint_timen("Closing all connections.")
sshlib.close_all_connections()
gp.lprint_timen("Connecting to "
+ open_connection_args['host'] + ".")
cix = sshlib.open_connection(**open_connection_args)
login_ssh(login_args)
continue
# We do not handle any other RuntimeErrors so we will raise the
# exception again.
sshlib.close_all_connections()
raise(execute_exception)
# If we get to this point, the command was executed.
break
if fork:
return
if rc != 0 and print_err:
gp.print_var(rc, 1)
if not print_out:
gp.print_var(stderr)
gp.print_var(stdout)
if print_out:
gp.printn(stderr + stdout)
if not ignore_err:
message = gp.sprint_error("The prior SSH"
+ " command returned a non-zero return"
+ " code:\n" + gp.sprint_var(rc, 1) + stderr
+ "\n")
BuiltIn().should_be_equal(rc, 0, message)
if open_connection_args['alias'] == "device_connection":
return stdout
return stdout, stderr, rc
def select_boot():
r"""
Select a boot test to be run based on our current state and return the
chosen boot type.
Description of arguments:
state The state of the machine.
"""
global boot_stack
gp.qprint_timen("Selecting a boot test.")
my_get_state()
stack_popped = 0
if len(boot_stack) > 0:
stack_popped = 1
gp.qprint_dashes()
gp.qprint_var(boot_stack)
gp.qprint_dashes()
skip_boot_printed = 0
while len(boot_stack) > 0:
boot_candidate = boot_stack.pop()
if stack_mode == 'normal':
break
else:
if st.compare_states(state, boot_table[boot_candidate]['end']):
if not skip_boot_printed:
gp.print_var(stack_mode)
gp.printn()
gp.print_timen("Skipping the following boot tests" +
" which are unnecessary since their" +
" required end states match the" +
" current machine state:")
skip_boot_printed = 1
gp.print_var(boot_candidate)
boot_candidate = ""
if boot_candidate == "":
gp.qprint_dashes()
gp.qprint_var(boot_stack)
gp.qprint_dashes()
return boot_candidate
if st.compare_states(state, boot_table[boot_candidate]['start']):
gp.qprint_timen("The machine state is valid for a '" +
boot_candidate + "' boot test.")
gp.qprint_dashes()
gp.qprint_var(boot_stack)
gp.qprint_dashes()
return boot_candidate
else:
gp.qprint_timen("The machine state does not match the required" +
" starting state for a '" + boot_candidate +
"' boot test:")
gp.print_varx("boot_table[" + boot_candidate + "][start]",
boot_table[boot_candidate]['start'], 1)
boot_stack.append(boot_candidate)
popped_boot = boot_candidate
# Loop through your list selecting a boot_candidates
boot_candidates = []
for boot_candidate in boot_list:
if st.compare_states(state, boot_table[boot_candidate]['start']):
if stack_popped:
if st.compare_states(boot_table[boot_candidate]['end'],
boot_table[popped_boot]['start']):
boot_candidates.append(boot_candidate)
else:
boot_candidates.append(boot_candidate)
if len(boot_candidates) == 0:
gp.qprint_timen("The user's boot list contained no boot tests" +
" which are valid for the current machine state.")
boot_candidate = default_power_on
if not st.compare_states(state, boot_table[default_power_on]['start']):
boot_candidate = default_power_off
boot_candidates.append(boot_candidate)
gp.qprint_timen("Using default '" + boot_candidate +
"' boot type to transition to valid state.")
gp.dprint_var(boot_candidates)
# Randomly select a boot from the candidate list.
boot = random.choice(boot_candidates)
return boot
def rprocess_plug_in_packages(plug_in_packages_list=None,
call_point="setup",
shell_rc="0x00000000",
stop_on_plug_in_failure=1,
stop_on_non_zero_rc=0,
release_type="obmc",
quiet=None,
debug=None,
return_history=False):
r"""
Call the external process_plug_in_packages.py to process the plug-in packages. Return the following:
rc The return code - 0 = PASS, 1 = FAIL.
shell_rc The shell return code returned by process_plug_in_packages.py.
failed_plug_in_name The failed plug in name (if any).
Description of arguments:
plug_in_packages_list A python list of plug-in directory paths.
call_point The call point program to be called for each plug-in package (e.g.
post_boot). This name should not include the "cp_" prefix.
shell_rc The user may supply a value other than zero to indicate an acceptable
non-zero return code. For example, if this value equals 0x00000200, it
means that for each plug-in call point that runs, a 0x00000200 will not
be counted as a failure.
stop_on_plug_in_failure If this parameter is set to 1, this program will stop and return non-zero
if the call point program from any plug-in directory fails. Conversely,
if it is set to false, this program will run the call point program from
each and every plug-in directory regardless of their return values.
Typical example cases where you'd want to run all plug-in call points
regardless of success or failure would be "cleanup" or "ffdc" call points.
stop_on_non_zero_rc If this parm is set to 1 and a plug-in call point program returns a valid
non-zero return code (see "shell_rc" parm above), this program will stop
processing and return 0 (success). Since this constitutes a successful
exit, this would normally be used where the caller wishes to stop
processing if one of the plug-in directory call point programs returns a
special value indicating that some special case has been found. An
example might be in calling some kind of "check_errl" call point program.
Such a call point program might return a 2 (i.e. 0x00000200) to indicate
that a given error log entry was found in an "ignore" list and is
therefore to be ignored. That being the case, no other "check_errl" call
point program would need to be called.
release_type The type of release being tested (e.g. "obmc", "op", "fips"). This
influences which integrated plug-ins are selected.
quiet If quiet is set to 1, this function will NOT write status messages to
stdout. This will default to the global quiet program parm or to 0.
debug If this parameter is set to 1, this function will print additional debug
information. This is mainly to be used by the developer of this
function. This will default to the global quiet program parm or to 0.
return_history In addition to rc, shell_rc and failed_plug_in_name, return a list
containing historical output that looks like the following:
history:
history[0]: #(CDT) 2018/10/30 12:25:49 - Running OBMC_Sample/cp_post_stack
"""
rc = 0
plug_in_packages_list = gp.get_var_value(plug_in_packages_list, [])
# If there are no plug-in packages to process, return successfully.
if len(plug_in_packages_list) == 0:
if return_history:
return 0, 0, "", []
else:
return 0, 0, ""
quiet = int(gp.get_var_value(quiet, 0))
debug = int(gp.get_var_value(debug, 0))
# Create string from list.
plug_in_dir_paths = ':'.join(plug_in_packages_list)
temp = tempfile.NamedTemporaryFile()
temp_file_path = temp.name
temp2 = tempfile.NamedTemporaryFile()
temp_properties_file_path = temp2.name
if debug:
os.environ["PERF_TRACE"] = "1"
debug_string = " --quiet=0"
else:
debug_string = ""
loc_shell_rc = 0
sub_cmd_buf = "process_plug_in_packages.py" + debug_string +\
" --call_point=" + call_point + " --allow_shell_rc=" +\
str(shell_rc) + " --stop_on_plug_in_failure=" +\
str(stop_on_plug_in_failure) + " --stop_on_non_zero_rc=" +\
str(stop_on_non_zero_rc) + " " + plug_in_dir_paths
if quiet:
cmd_buf = sub_cmd_buf + " > " + temp_file_path + " 2>&1"
else:
cmd_buf = "set -o pipefail ; " + sub_cmd_buf + " 2>&1 | tee " +\
temp_file_path
if debug:
gp.print_issuing(cmd_buf)
else:
gp.print_timen("Processing " + call_point +
" call point programs.")
sub_proc = subprocess.Popen(cmd_buf, shell=True, executable='/bin/bash')
sub_proc.communicate()
proc_plug_pkg_rc = sub_proc.returncode
if return_history:
# Get the "Running" statements from the output.
regex = " Running [^/]+/cp_"
cmd_buf = "egrep '" + regex + "' " + temp_file_path
_, history = gc.shell_cmd(cmd_buf,
quiet=(not debug),
print_output=0,
show_err=0,
ignore_err=1)
history = [x + "\n" for x in filter(None, history.split("\n"))]
else:
history = []
# As process_plug_in_packages.py help text states, it will print the values of failed_plug_in_name and
# shell_rc in the following format:
# failed_plug_in_name: <failed plug-in value, if any>
# shell_rc: <shell return code value of last call point program>
# We want to obtain those values from the output. To make the task simpler, we'll start by grepping the
# output for lines that might fit such a format:
# A valid bash variable against the left margin followed by...
# - A colon followed by...
# - Zero or more spaces
bash_var_regex = "[_[:alpha:]][_[:alnum:]]*"
regex = "^" + bash_var_regex + ":[ ]*"
cmd_buf = "egrep '" + regex + "' " + temp_file_path + " > " +\
temp_properties_file_path
gp.dprint_issuing(cmd_buf)
grep_rc = os.system(cmd_buf)
# Next we call my_parm_file to create a properties dictionary.
properties = gm.my_parm_file(temp_properties_file_path)
# Finally, we access the 2 values that we need.
shell_rc = int(properties.get('shell_rc', '0x0000000000000000'), 16)
failed_plug_in_name = properties.get('failed_plug_in_name', '')
if proc_plug_pkg_rc != 0:
if quiet:
os.system("cat " + temp_file_path + " >&2")
if grep_rc != 0:
gp.print_var(grep_rc, gp.hexa())
gp.print_var(proc_plug_pkg_rc, gp.hexa())
gp.print_timen("Re-cap of plug-in failures:")
gc.cmd_fnc_u("egrep -A 1 '^failed_plug_in_name:[ ]+' " +
temp_properties_file_path + " | egrep -v '^\\--'",
quiet=1,
show_err=0)
rc = 1
if return_history:
return rc, shell_rc, failed_plug_in_name, history
else:
return rc, shell_rc, failed_plug_in_name
def robot_cmd_fnc(robot_cmd_buf,
robot_jail=os.environ.get('ROBOT_JAIL', ''),
gzip=1):
r"""
Run the robot command string.
This function will set the various PATH variables correctly so that you
are running the proper version of all imported files, etc.
Description of argument(s):
robot_cmd_buf The complete robot command string.
robot_jail Indicates that this is to run in "robot
jail" meaning without visibility to any
apolloxxx import files, programs, etc.
gqip This indicates that the log, report and
output files produced by robot should be
gzipped to save space.
"""
if not gv.valid_value(robot_cmd_buf):
return False
# Set global variables to aid in cleanup with process_robot_output_files.
global gcr_last_robot_cmd_buf
global gcr_last_robot_rc
gcr_last_robot_cmd_buf = robot_cmd_buf
# Get globals set by init_robot_test_base_dir_path().
module = sys.modules["__main__"]
try:
ROBOT_TEST_BASE_DIR_PATH = getattr(module, "ROBOT_TEST_BASE_DIR_PATH")
except NameError:
init_robot_test_base_dir_path()
ROBOT_TEST_BASE_DIR_PATH = getattr(module, "ROBOT_TEST_BASE_DIR_PATH")
ROBOT_TEST_RUNNING_FROM_SB = \
gm.get_mod_global("ROBOT_TEST_RUNNING_FROM_SB")
if robot_jail == "":
if ROBOT_TEST_RUNNING_FROM_SB:
robot_jail = 0
else:
robot_jail = 1
robot_jail = int(robot_jail)
ROBOT_JAIL = os.environ.get('ROBOT_JAIL', '')
gp.dprint_vars(ROBOT_TEST_BASE_DIR_PATH, ROBOT_TEST_RUNNING_FROM_SB,
ROBOT_JAIL, robot_jail)
# Save PATH and PYTHONPATH to be restored later.
os.environ["SAVED_PYTHONPATH"] = os.environ.get("PYTHONPATH", "")
os.environ["SAVED_PATH"] = os.environ.get("PATH", "")
if robot_jail:
PYTHONPATH = ROBOT_TEST_BASE_DIR_PATH + "lib"
NEW_PATH_LIST = [ROBOT_TEST_BASE_DIR_PATH + "bin"]
# Coding special case to preserve python27_path.
python27_path = "/opt/rh/python27/root/usr/bin"
PATH_LIST = os.environ.get("PATH", "").split(":")
if python27_path in PATH_LIST:
NEW_PATH_LIST.append(python27_path)
NEW_PATH_LIST.extend(["/usr/local/sbin", "/usr/local/bin", "/usr/sbin",
"/usr/bin", "/sbin", "/bin"])
PATH = ":".join(NEW_PATH_LIST)
else:
PYTHONPATH = os.environ.get('PYTHONPATH', '') + ":" +\
ROBOT_TEST_BASE_DIR_PATH + "lib/"
PATH = os.environ.get('PATH', '') + ":" + ROBOT_TEST_BASE_DIR_PATH +\
"bin/"
os.environ['PYTHONPATH'] = PYTHONPATH
os.environ['PATH'] = PATH
gp.dprint_vars(PATH, PYTHONPATH)
os.environ['FFDC_DIR_PATH_STYLE'] = os.environ.get('FFDC_DIR_PATH_STYLE',
'1')
test_mode = getattr(module, "test_mode")
gp.qpissuing(robot_cmd_buf, test_mode)
if test_mode:
os.environ["PATH"] = os.environ.get("SAVED_PATH", "")
os.environ["PYTHONPATH"] = os.environ.get("SAVED_PYTHONPATH", "")
return True
if quiet:
DEVNULL = open(os.devnull, 'wb')
stdout = DEVNULL
else:
stdout = None
sub_proc = subprocess.Popen(robot_cmd_buf, stdout=stdout, shell=True)
sub_proc.communicate()
shell_rc = sub_proc.returncode
os.environ["PATH"] = os.environ.get("SAVED_PATH", "")
os.environ["PYTHONPATH"] = os.environ.get("SAVED_PYTHONPATH", "")
gcr_last_robot_rc = shell_rc
process_robot_output_files()
if shell_rc != 0:
hex = 1
gp.print_var(shell_rc, hex)
return False
return True
def robot_cmd_fnc(robot_cmd_buf,
robot_jail=os.environ.get('ROBOT_JAIL', ''),
quiet=None,
test_mode=0):
r"""
Run the robot command string.
This function will set the various PATH variables correctly so that you are running the proper version of
all imported files, etc.
Description of argument(s):
robot_cmd_buf The complete robot command string.
robot_jail Indicates that this is to run in "robot jail" meaning without visibility
to any apolloxxx import files, programs, etc.
test_mode If test_mode is set, this function will not actually run the command.
"""
quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0)))
gv.valid_value(robot_cmd_buf)
# Set global variables to aid in cleanup with process_robot_output_files.
global gcr_last_robot_cmd_buf
global gcr_last_robot_rc
gcr_last_robot_cmd_buf = robot_cmd_buf
# Get globals set by init_robot_test_base_dir_path().
module = sys.modules["__main__"]
try:
ROBOT_TEST_BASE_DIR_PATH = getattr(module, "ROBOT_TEST_BASE_DIR_PATH")
except NameError:
init_robot_test_base_dir_path()
ROBOT_TEST_BASE_DIR_PATH = getattr(module, "ROBOT_TEST_BASE_DIR_PATH")
ROBOT_TEST_RUNNING_FROM_SB = gm.get_mod_global(
"ROBOT_TEST_RUNNING_FROM_SB")
OPENBMCTOOL_DIR_PATH = gm.get_mod_global("OPENBMCTOOL_DIR_PATH")
if robot_jail == "":
if ROBOT_TEST_RUNNING_FROM_SB:
robot_jail = 0
else:
robot_jail = 1
robot_jail = int(robot_jail)
ROBOT_JAIL = os.environ.get('ROBOT_JAIL', '')
gp.dprint_vars(ROBOT_TEST_BASE_DIR_PATH, ROBOT_TEST_RUNNING_FROM_SB,
ROBOT_JAIL, robot_jail)
# Save PATH and PYTHONPATH to be restored later.
os.environ["SAVED_PYTHONPATH"] = os.environ.get("PYTHONPATH", "")
os.environ["SAVED_PATH"] = os.environ.get("PATH", "")
if robot_jail:
# Make sure required programs like python and robot can be found in the new restricted PATH.
required_programs = "python robot"
# It is expected that there will be a "python" program in the tool base bin path which is really a
# link to select_version. Ditto for "robot". Call each with the --print_only option to get the
# paths to the "real" programs.
cmd_buf = "for program in " + required_programs \
+ " ; do dirname $(${program} --print_only) ; done 2>/dev/null"
rc, out_buf = gc.shell_cmd(cmd_buf, quiet=1, print_output=0)
PYTHONPATH = ROBOT_TEST_BASE_DIR_PATH + "lib"
NEW_PATH_LIST = [ROBOT_TEST_BASE_DIR_PATH + "bin"]
NEW_PATH_LIST.extend(list(set(out_buf.rstrip("\n").split("\n"))))
NEW_PATH_LIST.extend([
"/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin",
"/sbin", "/bin",
OPENBMCTOOL_DIR_PATH.rstrip('/')
])
PATH = ":".join(NEW_PATH_LIST)
else:
PYTHONPATH = os.environ.get('PYTHONPATH', '') + ":" +\
ROBOT_TEST_BASE_DIR_PATH + "lib"
PATH = os.environ.get('PATH', '') + ":" + ROBOT_TEST_BASE_DIR_PATH +\
"bin" + ":" + OPENBMCTOOL_DIR_PATH.rstrip('/')
os.environ['PYTHONPATH'] = PYTHONPATH
os.environ['PATH'] = PATH
gp.dprint_vars(PATH, PYTHONPATH)
os.environ['FFDC_DIR_PATH_STYLE'] = os.environ.get('FFDC_DIR_PATH_STYLE',
'1')
gp.qpissuing(robot_cmd_buf, test_mode)
if test_mode:
os.environ["PATH"] = os.environ.get("SAVED_PATH", "")
os.environ["PYTHONPATH"] = os.environ.get("SAVED_PYTHONPATH", "")
return True
if quiet:
DEVNULL = open(os.devnull, 'wb')
stdout = DEVNULL
else:
stdout = None
sub_proc = subprocess.Popen(robot_cmd_buf, stdout=stdout, shell=True)
sub_proc.communicate()
shell_rc = sub_proc.returncode
os.environ["PATH"] = os.environ.get("SAVED_PATH", "")
os.environ["PYTHONPATH"] = os.environ.get("SAVED_PYTHONPATH", "")
gcr_last_robot_rc = shell_rc
process_robot_output_files()
if shell_rc != 0:
gp.print_var(shell_rc, gp.hexa())
return False
return True
def execute_ssh_command(cmd_buf,
open_connection_args={},
login_args={},
print_out=0,
print_err=0,
ignore_err=1,
fork=0,
quiet=None,
test_mode=None,
time_out=None):
r"""
Run the given command in an SSH session and return the stdout, stderr and
the return code.
If there is no open SSH connection, this function will connect and login.
Likewise, if the caller has not yet logged in to the connection, this
function will do the login.
NOTE: There is special handling when open_connection_args['alias'] equals
"device_connection".
- A write, rather than an execute_command, is done.
- Only stdout is returned (no stderr or rc).
- print_err, ignore_err and fork are not supported.
Description of arguments:
cmd_buf The command string to be run in an SSH
session.
open_connection_args A dictionary of arg names and values which
are legal to pass to the SSHLibrary
open_connection function as parms/args.
At a minimum, this should contain a 'host'
entry.
login_args A dictionary containing the key/value
pairs which are acceptable to the
SSHLibrary login function as parms/args.
At a minimum, this should contain a
'username' and a 'password' entry.
print_out If this is set, this function will print
the stdout/stderr generated by the shell
command.
print_err If show_err is set, this function will
print a standardized error report if the
shell command returns non-zero.
ignore_err Indicates that errors encountered on the
sshlib.execute_command are to be ignored.
fork Indicates that sshlib.start is to be used
rather than sshlib.execute_command.
quiet Indicates whether this function should run
the pissuing() function which prints an
"Issuing: <cmd string>" to stdout. This
defaults to the global quiet value.
test_mode If test_mode is set, this function will
not actually run the command. This
defaults to the global test_mode value.
time_out The amount of time to allow for the
execution of cmd_buf. A value of None
means that there is no limit to how long
the command may take.
"""
gp.lprint_executing()
# Obtain default values.
quiet = int(gp.get_var_value(quiet, 0))
test_mode = int(gp.get_var_value(test_mode, 0))
if not quiet:
gp.pissuing(cmd_buf, test_mode)
gp.lpissuing(cmd_buf, test_mode)
if test_mode:
return "", "", 0
global sshlib
max_exec_cmd_attempts = 2
# Look for existing SSH connection.
# Prepare a search connection dictionary.
search_connection_args = open_connection_args.copy()
# Remove keys that don't work well for searches.
search_connection_args.pop("timeout", None)
connection = find_connection(search_connection_args)
if connection:
gp.lprint_timen("Found the following existing connection:")
gp.lprintn(sprint_connection(connection))
if connection.alias == "":
index_or_alias = connection.index
else:
index_or_alias = connection.alias
gp.lprint_timen("Switching to existing connection: \"" +
str(index_or_alias) + "\".")
sshlib.switch_connection(index_or_alias)
else:
gp.lprint_timen("Connecting to " + open_connection_args['host'] + ".")
cix = sshlib.open_connection(**open_connection_args)
try:
login_ssh(login_args)
except Exception:
except_type, except_value, except_traceback = sys.exc_info()
rc = 1
stderr = str(except_value)
stdout = ""
max_exec_cmd_attempts = 0
for exec_cmd_attempt_num in range(1, max_exec_cmd_attempts + 1):
gp.lprint_var(exec_cmd_attempt_num)
try:
if fork:
sshlib.start_command(cmd_buf)
else:
if open_connection_args['alias'] == "device_connection":
stdout = sshlib.write(cmd_buf)
stderr = ""
rc = 0
else:
stdout, stderr, rc = \
func_timer.run(sshlib.execute_command,
cmd_buf,
return_stdout=True,
return_stderr=True,
return_rc=True,
time_out=time_out)
except Exception:
except_type, except_value, except_traceback = sys.exc_info()
gp.lprint_var(except_type)
gp.lprint_varx("except_value", str(except_value))
# This may be our last time through the retry loop, so setting
# return variables.
rc = 1
stderr = str(except_value)
stdout = ""
if except_type is exceptions.AssertionError and\
re.match(r"Connection not open", str(except_value)):
try:
login_ssh(login_args)
# Now we must continue to next loop iteration to retry the
# execute_command.
continue
except Exception:
except_type, except_value, except_traceback =\
sys.exc_info()
rc = 1
stderr = str(except_value)
stdout = ""
break
if (except_type is paramiko.ssh_exception.SSHException
and re.match(r"SSH session not active", str(except_value))) or\
(except_type is socket.error
and re.match(r"\[Errno 104\] Connection reset by peer",
str(except_value))) or\
(except_type is paramiko.ssh_exception.SSHException
and re.match(r"Timeout opening channel\.",
str(except_value))):
# Close and re-open a connection.
# Note: close_connection() doesn't appear to get rid of the
# connection. It merely closes it. Since there is a concern
# about over-consumption of resources, we use
# close_all_connections() which also gets rid of all
# connections.
gp.lprint_timen("Closing all connections.")
sshlib.close_all_connections()
gp.lprint_timen("Connecting to " +
open_connection_args['host'] + ".")
cix = sshlib.open_connection(**open_connection_args)
login_ssh(login_args)
continue
# We do not handle any other RuntimeErrors so we will raise the
# exception again.
sshlib.close_all_connections()
gp.lprintn(traceback.format_exc())
raise (except_value)
# If we get to this point, the command was executed.
break
if fork:
return
if rc != 0 and print_err:
gp.print_var(rc, gp.hexa())
if not print_out:
gp.print_var(stderr)
gp.print_var(stdout)
if print_out:
gp.printn(stderr + stdout)
if not ignore_err:
message = gp.sprint_error("The prior SSH" +
" command returned a non-zero return" +
" code:\n" + gp.sprint_var(rc, gp.hexa()) +
stderr + "\n")
BuiltIn().should_be_equal(rc, 0, message)
if open_connection_args['alias'] == "device_connection":
return stdout
return stdout, stderr, rc
def execute_ssh_command(cmd_buf,
open_connection_args={},
login_args={},
print_out=0,
print_err=0,
ignore_err=1,
fork=0,
quiet=None,
test_mode=None):
r"""
Run the given command in an SSH session and return the stdout, stderr and
the return code.
If there is no open SSH connection, this function will connect and login.
Likewise, if the caller has not yet logged in to the connection, this
function will do the login.
Description of arguments:
cmd_buf The command string to be run in an SSH
session.
open_connection_args A dictionary of arg names and values which
are legal to pass to the SSHLibrary
open_connection function as parms/args.
At a minimum, this should contain a 'host'
entry.
login_args A dictionary containing the key/value
pairs which are acceptable to the
SSHLibrary login function as parms/args.
At a minimum, this should contain a
'username' and a 'password' entry.
print_out If this is set, this function will print
the stdout/stderr generated by the shell
command.
print_err If show_err is set, this function will
print a standardized error report if the
shell command returns non-zero.
ignore_err Indicates that errors encountered on the
sshlib.execute_command are to be ignored.
fork Indicates that sshlib.start is to be used
rather than sshlib.execute_command.
quiet Indicates whether this function should run
the pissuing() function which prints an
"Issuing: <cmd string>" to stdout. This
defaults to the global quiet value.
test_mode If test_mode is set, this function will
not actually run the command. This
defaults to the global test_mode value.
"""
gp.dprint_executing()
# Obtain default values.
quiet = int(gp.get_var_value(quiet, 0))
test_mode = int(gp.get_var_value(test_mode, 0))
if not quiet:
gp.pissuing(cmd_buf, test_mode)
if test_mode:
return "", "", 0
global sshlib
# Look for existing SSH connection.
# Prepare a search connection dictionary.
search_connection_args = open_connection_args.copy()
# Remove keys that don't work well for searches.
search_connection_args.pop("timeout", None)
connection = find_connection(search_connection_args)
if connection:
gp.dprint_timen("Found the following existing connection:")
gp.dprintn(sprint_connection(connection))
if connection.alias == "":
index_or_alias = connection.index
else:
index_or_alias = connection.alias
gp.dprint_timen("Switching to existing connection: \"" +
str(index_or_alias) + "\".")
sshlib.switch_connection(index_or_alias)
else:
gp.dprint_timen("Connecting to " + open_connection_args['host'] + ".")
cix = sshlib.open_connection(**open_connection_args)
login_ssh(login_args)
max_exec_cmd_attempts = 2
for exec_cmd_attempt_num in range(1, max_exec_cmd_attempts + 1):
gp.dprint_var(exec_cmd_attempt_num)
try:
if fork:
sshlib.start_command(cmd_buf)
else:
stdout, stderr, rc = sshlib.execute_command(cmd_buf,
return_stdout=True,
return_stderr=True,
return_rc=True)
except Exception as execute_exception:
except_type, except_value, except_traceback = sys.exc_info()
gp.dprint_var(except_type)
gp.dprint_varx("except_value", str(except_value))
if except_type is exceptions.AssertionError and\
re.match(r"Connection not open", str(except_value)):
login_ssh(login_args)
# Now we must continue to next loop iteration to retry the
# execute_command.
continue
if except_type is paramiko.ssh_exception.SSHException and\
re.match(r"SSH session not active", str(except_value)):
# Close and re-open a connection.
sshlib.close_connection()
gp.dprint_timen("Connecting to " +
open_connection_args['host'] + ".")
cix = sshlib.open_connection(**open_connection_args)
login_ssh(login_args)
continue
# We do not handle any other RuntimeErrors so we will raise the
# exception again.
raise(execute_exception)
# If we get to this point, the command was executed.
break
if fork:
return
if rc != 0 and print_err:
gp.print_var(rc, 1)
if not print_out:
gp.print_var(stderr)
gp.print_var(stdout)
if print_out:
gp.printn(stderr + stdout)
if not ignore_err:
message = gp.sprint_error("The prior SSH" +
" command returned a non-zero return" +
" code:\n" + gp.sprint_var(rc, 1) + stderr +
"\n")
BuiltIn().should_be_equal(rc, 0, message)
return stdout, stderr, rc
