def obmc_boot_test_teardown():
r"""
Clean up after the Main keyword.
"""
if cp_setup_called:
plug_in_setup()
rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
call_point='cleanup', stop_on_plug_in_failure=0)
if 'boot_results_file_path' in globals():
# Save boot_results object to a file in case it is needed again.
gp.qprint_timen("Saving boot_results to the following path.")
gp.qprint_var(boot_results_file_path)
pickle.dump(boot_results, open(boot_results_file_path, 'wb'),
pickle.HIGHEST_PROTOCOL)
global save_stack
# Restore any global values saved on the save_stack.
for parm_name in main_func_parm_list:
# Get the parm_value if it was saved on the stack.
try:
parm_value = save_stack.pop(parm_name)
except:
# If it was not saved, no further action is required.
continue
# Restore the saved value.
cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
"}\", parm_value)"
gp.dpissuing(cmd_buf)
exec(cmd_buf)
gp.dprintn(save_stack.sprint_obj())
def obmc_boot_test_teardown():
r"""
Clean up after the Main keyword.
"""
if cp_setup_called:
plug_in_setup()
rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
call_point='cleanup', stop_on_plug_in_failure=0)
if 'boot_results_file_path' in globals():
# Save boot_results object to a file in case it is needed again.
gp.qprint_timen("Saving boot_results to the following path.")
gp.qprint_var(boot_results_file_path)
pickle.dump(boot_results, open(boot_results_file_path, 'wb'),
pickle.HIGHEST_PROTOCOL)
global save_stack
# Restore any global values saved on the save_stack.
for parm_name in main_func_parm_list:
# Get the parm_value if it was saved on the stack.
try:
parm_value = save_stack.pop(parm_name)
except BaseException:
# If it was not saved, no further action is required.
continue
# Restore the saved value.
cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
"}\", parm_value)"
gp.dpissuing(cmd_buf)
exec(cmd_buf)
gp.dprintn(save_stack.sprint_obj())
def login_ssh(login_args={},
max_login_attempts=5):
r"""
Login on the latest open SSH connection. Retry on failure up to
max_login_attempts.
The caller is responsible for making sure there is an open SSH connection.
Description of argument(s):
login_args A dictionary containing the key/value
pairs which are acceptable to the
SSHLibrary login function as parms/args.
At a minimum, this should contain a
'username' and a 'password' entry.
max_login_attempts The max number of times to try logging in
(in the event of login failures).
"""
global sshlib
# Get connection data for debug output.
connection = sshlib.get_connection()
gp.dprintn(sprint_connection(connection))
for login_attempt_num in range(1, max_login_attempts + 1):
gp.dprint_timen("Logging in to " + connection.host + ".")
gp.dprint_var(login_attempt_num)
try:
out_buf = sshlib.login(**login_args)
except Exception as login_exception:
# Login will sometimes fail if the connection is new.
except_type, except_value, except_traceback = sys.exc_info()
gp.dprint_var(except_type)
gp.dprint_varx("except_value", str(except_value))
if except_type is paramiko.ssh_exception.SSHException and\
re.match(r"No existing session", str(except_value)):
continue
else:
# We don't tolerate any other error so break from loop and
# re-raise exception.
break
# If we get to this point, the login has worked and we can return.
gp.dpvar(out_buf)
return
# If we get to this point, the login has failed on all attempts so the
# exception will be raised again.
raise(login_exception)
def obmc_boot_test_py(loc_boot_stack=None,
loc_stack_mode=None,
loc_quiet=None):
r"""
Do main program processing.
"""
global save_stack
# Process function parms.
for parm_name in main_func_parm_list:
# Get parm's value.
cmd_buf = "parm_value = loc_" + parm_name
exec(cmd_buf)
gp.dpvar(parm_name)
gp.dpvar(parm_value)
if parm_value is None:
# Parm was not specified by the calling function so set it to its
# corresponding global value.
cmd_buf = "loc_" + parm_name + " = BuiltIn().get_variable_value" +\
"(\"${" + parm_name + "}\")"
gp.dpissuing(cmd_buf)
exec(cmd_buf)
else:
# Save the global value on a stack.
cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\
parm_name + "}\"), \"" + parm_name + "\")"
gp.dpissuing(cmd_buf)
exec(cmd_buf)
# Set the global value to the passed value.
cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
"}\", loc_" + parm_name + ")"
gp.dpissuing(cmd_buf)
exec(cmd_buf)
gp.dprintn(save_stack.sprint_obj())
setup()
if ffdc_only:
gp.qprint_timen("Caller requested ffdc_only.")
pre_boot_plug_in_setup()
grk.run_key_u("my_ffdc")
return
# Process caller's boot_stack.
while (len(boot_stack) > 0):
test_loop_body()
gp.qprint_timen("Finished processing stack.")
# Process caller's boot_list.
if len(boot_list) > 0:
for ix in range(1, max_num_tests + 1):
test_loop_body()
gp.qprint_timen("Completed all requested boot tests.")
boot_pass, boot_fail = boot_results.return_total_pass_fail()
if boot_fail > boot_fail_threshold:
error_message = "Boot failures exceed the boot failure" +\
" threshold:\n" +\
gp.sprint_var(boot_fail) +\
gp.sprint_var(boot_fail_threshold)
BuiltIn().fail(gp.sprint_error(error_message))
def obmc_boot_test_py(loc_boot_stack=None,
loc_stack_mode=None,
loc_quiet=None):
r"""
Do main program processing.
"""
global save_stack
ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']})
gp.dprintn()
# Process function parms.
for parm_name in main_func_parm_list:
# Get parm's value.
parm_value = eval("loc_" + parm_name)
gp.dpvars(parm_name, parm_value)
if parm_value is not None:
# Save the global value on a stack.
cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\
parm_name + "}\"), \"" + parm_name + "\")"
gp.dpissuing(cmd_buf)
exec(cmd_buf)
# Set the global value to the passed value.
cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
"}\", loc_" + parm_name + ")"
gp.dpissuing(cmd_buf)
exec(cmd_buf)
gp.dprintn(save_stack.sprint_obj())
setup()
init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail()
if ffdc_only:
gp.qprint_timen("Caller requested ffdc_only.")
if do_pre_boot_plug_in_setup:
pre_boot_plug_in_setup()
grk.run_key_u("my_ffdc")
return
if delete_errlogs:
# print error logs before delete
status, error_logs = grk.run_key_u("Get Error Logs")
pels = pel.peltool("-l", ignore_err=1)
log.print_error_logs(error_logs, "AdditionalData Message Severity")
gp.qprint_var(pels)
# Delete errlogs prior to doing any boot tests.
grk.run_key(delete_errlogs_cmd, ignore=1)
grk.run_key(delete_bmcdump_cmd, ignore=1)
# Process caller's boot_stack.
while (len(boot_stack) > 0):
test_loop_body()
gp.qprint_timen("Finished processing stack.")
post_stack()
# Process caller's boot_list.
if len(boot_list) > 0:
for ix in range(1, max_num_tests + 1):
test_loop_body()
gp.qprint_timen("Completed all requested boot tests.")
boot_pass, boot_fail = boot_results.return_total_pass_fail()
new_fail = boot_fail - init_boot_fail
if new_fail > boot_fail_threshold:
error_message = "Boot failures exceed the boot failure" +\
" threshold:\n" +\
gp.sprint_var(new_fail) +\
gp.sprint_var(boot_fail_threshold)
BuiltIn().fail(gp.sprint_error(error_message))
def obmc_boot_test_py(loc_boot_stack=None,
loc_stack_mode=None,
loc_quiet=None):
r"""
Do main program processing.
"""
global save_stack
# Process function parms.
for parm_name in main_func_parm_list:
# Get parm's value.
cmd_buf = "parm_value = loc_" + parm_name
exec(cmd_buf)
gp.dpvar(parm_name)
gp.dpvar(parm_value)
if parm_value is None:
# Parm was not specified by the calling function so set it to its
# corresponding global value.
cmd_buf = "loc_" + parm_name + " = BuiltIn().get_variable_value" +\
"(\"${" + parm_name + "}\")"
gp.dpissuing(cmd_buf)
exec(cmd_buf)
else:
# Save the global value on a stack.
cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\
parm_name + "}\"), \"" + parm_name + "\")"
gp.dpissuing(cmd_buf)
exec(cmd_buf)
# Set the global value to the passed value.
cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
"}\", loc_" + parm_name + ")"
gp.dpissuing(cmd_buf)
exec(cmd_buf)
gp.dprintn(save_stack.sprint_obj())
setup()
init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail()
if ffdc_only:
gp.qprint_timen("Caller requested ffdc_only.")
pre_boot_plug_in_setup()
grk.run_key_u("my_ffdc")
return
# Process caller's boot_stack.
while (len(boot_stack) > 0):
test_loop_body()
gp.qprint_timen("Finished processing stack.")
# Process caller's boot_list.
if len(boot_list) > 0:
for ix in range(1, max_num_tests + 1):
test_loop_body()
gp.qprint_timen("Completed all requested boot tests.")
boot_pass, boot_fail = boot_results.return_total_pass_fail()
new_fail = boot_fail - init_boot_fail
if new_fail > boot_fail_threshold:
error_message = "Boot failures exceed the boot failure" +\
" threshold:\n" +\
gp.sprint_var(new_fail) +\
gp.sprint_var(boot_fail_threshold)
BuiltIn().fail(gp.sprint_error(error_message))
def execute_ssh_command(cmd_buf,
open_connection_args={},
login_args={},
print_out=0,
print_err=0,
ignore_err=1,
fork=0,
quiet=None,
test_mode=None):
r"""
Run the given command in an SSH session and return the stdout, stderr and
the return code.
If there is no open SSH connection, this function will connect and login.
Likewise, if the caller has not yet logged in to the connection, this
function will do the login.
Description of arguments:
cmd_buf The command string to be run in an SSH
session.
open_connection_args A dictionary of arg names and values which
are legal to pass to the SSHLibrary
open_connection function as parms/args.
At a minimum, this should contain a 'host'
entry.
login_args A dictionary containing the key/value
pairs which are acceptable to the
SSHLibrary login function as parms/args.
At a minimum, this should contain a
'username' and a 'password' entry.
print_out If this is set, this function will print
the stdout/stderr generated by the shell
command.
print_err If show_err is set, this function will
print a standardized error report if the
shell command returns non-zero.
ignore_err Indicates that errors encountered on the
sshlib.execute_command are to be ignored.
fork Indicates that sshlib.start is to be used
rather than sshlib.execute_command.
quiet Indicates whether this function should run
the pissuing() function which prints an
"Issuing: <cmd string>" to stdout. This
defaults to the global quiet value.
test_mode If test_mode is set, this function will
not actually run the command. This
defaults to the global test_mode value.
"""
gp.dprint_executing()
# Obtain default values.
quiet = int(gp.get_var_value(quiet, 0))
test_mode = int(gp.get_var_value(test_mode, 0))
if not quiet:
gp.pissuing(cmd_buf, test_mode)
if test_mode:
return "", "", 0
global sshlib
# Look for existing SSH connection.
# Prepare a search connection dictionary.
search_connection_args = open_connection_args.copy()
# Remove keys that don't work well for searches.
search_connection_args.pop("timeout", None)
connection = find_connection(search_connection_args)
if connection:
gp.dprint_timen("Found the following existing connection:")
gp.dprintn(sprint_connection(connection))
if connection.alias == "":
index_or_alias = connection.index
else:
index_or_alias = connection.alias
gp.dprint_timen("Switching to existing connection: \"" +
str(index_or_alias) + "\".")
sshlib.switch_connection(index_or_alias)
else:
gp.dprint_timen("Connecting to " + open_connection_args['host'] + ".")
cix = sshlib.open_connection(**open_connection_args)
login_ssh(login_args)
max_exec_cmd_attempts = 2
for exec_cmd_attempt_num in range(1, max_exec_cmd_attempts + 1):
gp.dprint_var(exec_cmd_attempt_num)
try:
if fork:
sshlib.start_command(cmd_buf)
else:
stdout, stderr, rc = sshlib.execute_command(cmd_buf,
return_stdout=True,
return_stderr=True,
return_rc=True)
except Exception as execute_exception:
except_type, except_value, except_traceback = sys.exc_info()
gp.dprint_var(except_type)
gp.dprint_varx("except_value", str(except_value))
if except_type is exceptions.AssertionError and\
re.match(r"Connection not open", str(except_value)):
login_ssh(login_args)
# Now we must continue to next loop iteration to retry the
# execute_command.
continue
if (except_type is paramiko.ssh_exception.SSHException and
re.match(r"SSH session not active", str(except_value))) or\
(except_type is socket.error and
re.match(r"\[Errno 104\] Connection reset by peer",
str(except_value))):
# Close and re-open a connection.
# Note: close_connection() doesn't appear to get rid of the
# connection. It merely closes it. Since there is a concern
# about over-consumption of resources, we use
# close_all_connections() which also gets rid of all
# connections.
gp.dprint_timen("Closing all connections.")
sshlib.close_all_connections()
gp.dprint_timen("Connecting to " +
open_connection_args['host'] + ".")
cix = sshlib.open_connection(**open_connection_args)
login_ssh(login_args)
continue
# We do not handle any other RuntimeErrors so we will raise the
# exception again.
raise (execute_exception)
# If we get to this point, the command was executed.
break
if fork:
return
if rc != 0 and print_err:
gp.print_var(rc, 1)
if not print_out:
gp.print_var(stderr)
gp.print_var(stdout)
if print_out:
gp.printn(stderr + stdout)
if not ignore_err:
message = gp.sprint_error("The prior SSH" +
" command returned a non-zero return" +
" code:\n" + gp.sprint_var(rc, 1) + stderr +
"\n")
BuiltIn().should_be_equal(rc, 0, message)
return stdout, stderr, rc
