def setup():
r"""
Do general program setup tasks.
"""
global cp_setup_called
gp.qprintn()
robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
# If we can't find process_plug_in_packages.py, ssh_pw or
# validate_plug_ins.py, then we don't have our repo bin in PATH.
shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" +
" ssh_pw validate_plug_ins.py", quiet=1,
print_output=0, show_err=0)
if shell_rc != 0:
os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "")
# Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
if robot_pgm_dir_path not in sys.path:
sys.path.append(robot_pgm_dir_path)
PYTHONPATH = os.environ.get("PYTHONPATH", "")
if PYTHONPATH == "":
os.environ['PYTHONPATH'] = robot_pgm_dir_path
else:
os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH
validate_parms()
grp.rqprint_pgm_header()
grk.run_key("Set BMC Power Policy RESTORE_LAST_STATE")
initial_plug_in_setup()
plug_in_setup()
rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
call_point='setup')
if rc != 0:
error_message = "Plug-in setup failed.\n"
grp.rprint_error_report(error_message)
BuiltIn().fail(error_message)
# Setting cp_setup_called lets our Teardown know that it needs to call
# the cleanup plug-in call point.
cp_setup_called = 1
# Keyword "FFDC" will fail if TEST_MESSAGE is not set.
BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
# FFDC_LOG_PATH is used by "FFDC" keyword.
BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
# Also printed by FFDC.
global host_name
global host_ip
host = socket.gethostname()
host_name, host_ip = gm.get_host_name_ip(host)
gp.dprint_var(boot_table, 1)
gp.dprint_var(boot_lists)
def verify_image_not_in_bmc_uploads_dir(image_version, timeout=3):
r"""
Check that an image with the given version is not unpacked inside of the
BMCs image uploads directory. If no image is found, retry every 30 seconds
until the given timeout is hit, in case the BMC takes time
unpacking the image.
Description of argument(s):
image_version The version of the image to look for on the BMC.
timeout How long, in minutes, to try to find an image on the BMC.
Default is 3 minutes.
"""
keyword.run_key('Open Connection And Log In')
for i in range(timeout * 2):
stat, grep_res = keyword.run_key('Execute Command On BMC ' + 'ls ' +
var.IMAGE_UPLOAD_DIR_PATH +
'*/MANIFEST 2>/dev/null ' +
'| xargs grep -rl "version=' +
image_version + '"')
image_dir = os.path.dirname(grep_res.split('\n')[0])
if '' != image_dir:
keyword.run_key('Execute Command On BMC rm -rf ' + image_dir)
BuiltIn().fail('Found invalid BMC Image: ' + image_dir)
time.sleep(30)
def scp_dumps(targ_dir_path, targ_file_prefix="", dump_dict=None, quiet=None):
r"""
SCP all dumps from the BMC to the indicated directory on the local system
and return a list of the new files.
Description of argument(s):
targ_dir_path The path of the directory to receive the
dump files.
targ_file_prefix Prefix which will be pre-pended to each
target file's name.
dump_dict A dump dictionary such as the one returned
by get_dump_dict. If this value is None,
this function will call get_dump_dict on
the caller's behalf.
quiet If quiet is set to 1, this function will
NOT write status messages to stdout.
"""
targ_dir_path = gm.add_trailing_slash(targ_dir_path)
if dump_dict is None:
dump_dict = get_dump_dict(quiet=quiet)
status, ret_values = grk.run_key("Open Connection for SCP", quiet=quiet)
dump_file_list = []
for dump_id, source_file_path in dump_dict.items():
targ_file_path = targ_dir_path + targ_file_prefix \
+ os.path.basename(source_file_path)
status, ret_values = grk.run_key("scp.Get File " + source_file_path +
" " + targ_file_path,
quiet=quiet)
dump_file_list.append(targ_file_path)
return dump_file_list
def delete_all_pnor_images():
r"""
Delete all PNOR images from the BMC.
"""
keyword.run_key("Initiate Host PowerOff")
status, images = keyword.run_key("Get Software Objects " +
var.VERSION_PURPOSE_HOST)
for image_name in images:
keyword.run_key("Delete Image And Verify " + image_name + " " +
var.VERSION_PURPOSE_HOST)
def delete_all_pnor_images():
r"""
Delete all PNOR images from the BMC.
"""
status, images = keyword.run_key("Get Software Objects "
+ var.VERSION_PURPOSE_HOST)
for image_name in images:
BuiltIn().log_to_console(image_name)
# Delete twice, in case the image is in the /tmp/images directory
keyword.run_key("Call Method " + image_name
+ " delete data={\"data\":[]}")
keyword.run_key("Call Method " + image_name
+ " delete data={\"data\":[]}")
def delete_all_pnor_images():
r"""
Delete all PNOR images from the BMC.
"""
status, images = keyword.run_key("Get Software Objects " +
var.VERSION_PURPOSE_HOST)
for image_name in images:
BuiltIn().log_to_console(image_name)
# Delete twice, in case the image is in the /tmp/images directory
keyword.run_key("Call Method " + image_name +
" delete data={\"data\":[]}")
keyword.run_key("Call Method " + image_name +
" delete data={\"data\":[]}")
def verify_image_upload():
r"""
Verify the image was uploaded correctly and that it created
a valid d-bus object
"""
image_version = BuiltIn().get_variable_value("${image_version}")
image_path = get_image_path(image_version)
image_version_id = image_path.split("/")[-2]
BuiltIn().set_global_variable("${version_id}", image_version_id)
grk.run_key_u("Open Connection And Log In")
image_purpose = get_image_purpose(image_path + "MANIFEST")
if (image_purpose == var.VERSION_PURPOSE_BMC
or image_purpose == var.VERSION_PURPOSE_HOST):
uri = var.SOFTWARE_VERSION + image_version_id
status, ret_values =\
grk.run_key("Read Attribute " + uri + " Activation")
if ((ret_values == var.READY) or (ret_values == var.INVALID)
or (ret_values == var.ACTIVE)):
return True
else:
gp.print_var(ret_values)
return False
else:
gp.print_var(image_purpose)
return False
def get_image_path(image_version):
r"""
Query the upload image dir for the presence of image matching
the version that was read from the MANIFEST before uploading
the image. Based on the purpose verify the activation object
exists and is either READY or INVALID.
Description of argument(s):
image_version The version of the image that should match one
of the images in the upload dir.
"""
keyword.run_key_u("Open Connection And Log In")
status, image_list =\
keyword.run_key("Execute Command On BMC ls -d "
+ var.IMAGE_UPLOAD_DIR_PATH + "*/")
image_list = image_list.split("\n")
retry = 0
while (retry < 10):
for i in range(0, len(image_list)):
version = get_image_version(image_list[i] + "MANIFEST")
if (version == image_version):
return image_list[i]
time.sleep(10)
retry += 1
def get_non_running_bmc_software_object():
r"""
Get the URI to a BMC image from software that is not running on the BMC.
"""
# Get the version of the image currently running on the BMC.
_, cur_img_version = keyword.run_key("Get BMC Version")
# Remove the surrounding double quotes from the version.
cur_img_version = cur_img_version.replace('"', '')
_, images = keyword.run_key("Read Properties " +
var.SOFTWARE_VERSION_URI + "enumerate")
for image_name in images:
_, image_properties = keyword.run_key("Get Host Software Property " +
image_name)
if image_properties['Purpose'] != var.VERSION_PURPOSE_HOST \
and image_properties['Version'] != cur_img_version:
return image_name
BuiltIn().fail("Did not find any non-running BMC images.")
def get_non_running_bmc_software_object():
r"""
Get the URI to a BMC image from software that is not running on the BMC.
"""
# Get the version of the image currently running on the BMC.
_, cur_img_version = keyword.run_key("Get BMC Version")
# Remove the surrounding double quotes from the version.
cur_img_version = cur_img_version.replace('"', '')
_, images = keyword.run_key("Read Properties "
+ var.SOFTWARE_VERSION_URI + "enumerate")
for image_name in images:
_, image_properties = keyword.run_key(
"Get Host Software Property " + image_name)
if 'Purpose' in image_properties and 'Version' in image_properties \
and image_properties['Purpose'] != var.VERSION_PURPOSE_HOST \
and image_properties['Version'] != cur_img_version:
return image_name
BuiltIn().fail("Did not find any non-running BMC images.")
def get_image_purpose(file_path):
r"""
Read the file for a purpose object.
Description of argument(s):
file_path The path to a file that holds the image purpose.
"""
keyword.run_key_u("Open Connection And Log In")
status, ret_values =\
keyword.run_key("Execute Command On BMC cat "
+ file_path + " | grep \"purpose=\"", ignore=1)
return ret_values.split("=")[-1]
def get_image_version(file_path):
r"""
Read the file for a version object.
Description of argument(s):
file_path The path to a file that holds the image version.
"""
grk.run_key_u("Open Connection And Log In")
status, ret_values =\
grk.run_key("Execute Command On BMC cat "
+ file_path + " | grep \"version=\"", ignore=1)
return (ret_values.split("\n")[0]).split("=")[-1]
def verify_image_not_in_bmc_uploads_dir(image_version):
r"""
Check that an image with the given version is not unpacked inside of the
BMCs image uploads directory. If no image is found, retry every 30 seconds
for 3 minutes in case the BMC takes time unpacking the image.
Description of argument(s):
image_version The version of the image to look for on the BMC.
"""
grk.run_key('Open Connection And Log In')
upload_dir_path = BuiltIn().get_variable_value("${UPLOAD_DIR_PATH}")
for i in range(6):
stat, grep_res = grk.run_key('Execute Command On BMC ' + 'ls ' +
upload_dir_path +
'*/MANIFEST 2>/dev/null ' +
'| xargs grep -rl "version=' +
image_version + '"')
image_dir = os.path.dirname(grep_res.split('\n')[0])
if '' != image_dir:
grk.run_key('Execute Command On BMC rm -rf ' + image_dir)
BuiltIn().fail('Found invalid BMC Image: ' + image_dir)
time.sleep(30)
def verify_no_duplicate_image_priorities(image_purpose):
r"""
Check that there are no active images with the same purpose and priority.
Description of argument(s):
image_purpose The purpose that images must have to be
checked for priority duplicates.
"""
taken_priorities = {}
_, image_names = keyword.run_key("Get Software Objects " +
"version_type=" + image_purpose)
for image_name in image_names:
_, image = keyword.run_key("Get Host Software Property " + image_name)
if image["Activation"] != var.ACTIVE:
continue
image_priority = image["Priority"]
if image_priority in taken_priorities:
BuiltIn().fail(
"Found active images with the same priority.\n" +
gp.sprint_vars(image, taken_priorities[image_priority]))
taken_priorities[image_priority] = image
def verify_no_duplicate_image_priorities(image_purpose):
r"""
Check that there are no active images with the same purpose and priority.
Description of argument(s):
image_purpose The purpose that images must have to be
checked for priority duplicates.
"""
taken_priorities = {}
_, image_names = keyword.run_key("Get Software Objects "
+ "version_type=" + image_purpose)
for image_name in image_names:
_, image = keyword.run_key("Get Host Software Property " + image_name)
if image["Activation"] != var.ACTIVE:
continue
image_priority = image["Priority"]
if image_priority in taken_priorities:
BuiltIn().fail("Found active images with the same priority.\n"
+ gp.sprint_vars(image,
taken_priorities[image_priority]))
taken_priorities[image_priority] = image
def scp_dumps(targ_dir_path,
targ_file_prefix="",
dump_dict=None,
quiet=None):
r"""
SCP all dumps from the BMC to the indicated directory on the local system
and return a list of the new files.
Description of argument(s):
targ_dir_path The path of the directory to receive the
dump files.
targ_file_prefix Prefix which will be pre-pended to each
target file's name.
dump_dict A dump dictionary such as the one returned
by get_dump_dict. If this value is None,
this function will call get_dump_dict on
the caller's behalf.
quiet If quiet is set to 1, this function will
NOT write status messages to stdout.
"""
targ_dir_path = gm.add_trailing_slash(targ_dir_path)
if dump_dict is None:
dump_dict = get_dump_dict(quiet=quiet)
status, ret_values = grk.run_key("Open Connection for SCP", quiet=quiet)
dump_file_list = []
for dump_id, source_file_path in dump_dict.iteritems():
targ_file_path = targ_dir_path + targ_file_prefix \
+ os.path.basename(source_file_path)
status, ret_values = grk.run_key("scp.Get File " + source_file_path
+ " " + targ_file_path, quiet=quiet)
dump_file_list.append(targ_file_path)
return dump_file_list
def get_latest_file(dir_path):
r"""
Get the path to the latest uploaded file.
Description of argument(s):
dir_path Path to the dir from which the name of the last
updated file or folder will be returned to the
calling function.
"""
keyword.run_key_u("Open Connection And Log In")
status, ret_values =\
keyword.run_key("Execute Command On BMC cd " + dir_path
+ "; stat -c '%Y %n' * | sort -k1,1nr | head -n 1", ignore=1)
return ret_values.split(" ")[-1]
def get_boot_state(self):
r"""
Return the system state as a tuple of bmc, chassis, host state,
BootProgress and OperatingSystemState.
"""
status, state = keyword.run_key("Read Properties " +
var.SYSTEM_STATE_URI + "enumerate")
bmc_state = state[var.SYSTEM_STATE_URI + 'bmc0']['CurrentBMCState']
chassis_state = \
state[var.SYSTEM_STATE_URI + 'chassis0']['CurrentPowerState']
host_state = state[var.SYSTEM_STATE_URI + 'host0']['CurrentHostState']
boot_state = state[var.SYSTEM_STATE_URI + 'host0']['BootProgress']
os_state = \
state[var.SYSTEM_STATE_URI + 'host0']['OperatingSystemState']
return (str(bmc_state), str(chassis_state), str(host_state),
str(boot_state), str(os_state))
def get_boot_state(self):
r"""
Return the system state as a tuple of bmc, chassis, host state,
BootProgress and OperatingSystemState.
"""
status, state = keyword.run_key("Read Properties "
+ var.SYSTEM_STATE_URI + "enumerate")
bmc_state = state[var.SYSTEM_STATE_URI + 'bmc0']['CurrentBMCState']
chassis_state = \
state[var.SYSTEM_STATE_URI + 'chassis0']['CurrentPowerState']
host_state = state[var.SYSTEM_STATE_URI + 'host0']['CurrentHostState']
boot_state = state[var.SYSTEM_STATE_URI + 'host0']['BootProgress']
os_state = \
state[var.SYSTEM_STATE_URI + 'host0']['OperatingSystemState']
return (str(bmc_state),
str(chassis_state),
str(host_state),
str(boot_state),
str(os_state))
def verify_image_upload(image_version,
timeout=3):
r"""
Verify the image was uploaded correctly and that it created
a valid d-bus object. If the first check for the image
fails, try again until we reach the timeout.
Description of argument(s):
image_version The version from the image's manifest file
(e.g. "IBM-witherspoon-redbud-ibm-OP9_v1.17_1.68").
timeout How long, in minutes, to keep trying to find the
image on the BMC. Default is 3 minutes.
"""
image_path = get_image_path(image_version)
image_version_id = image_path.split("/")[-2]
keyword.run_key_u("Open Connection And Log In")
image_purpose = get_image_purpose(image_path + "MANIFEST")
if (image_purpose == var.VERSION_PURPOSE_BMC or
image_purpose == var.VERSION_PURPOSE_HOST):
uri = var.SOFTWARE_VERSION_URI + image_version_id
ret_values = ""
for itr in range(timeout * 2):
status, ret_values = \
keyword.run_key("Read Attribute " + uri + " Activation")
if ((ret_values == var.READY) or (ret_values == var.INVALID)
or (ret_values == var.ACTIVE)):
return True, image_version_id
else:
time.sleep(30)
# If we exit the for loop, the timeout has been reached
gp.print_var(ret_values)
return False, None
else:
gp.print_var(image_purpose)
return False, None
def verify_image_upload(image_version,
timeout=3):
r"""
Verify the image was uploaded correctly and that it created
a valid d-bus object. If the first check for the image
fails, try again until we reach the timeout.
Description of argument(s):
image_version The version from the image's manifest file
(e.g. "v2.2-253-g00050f1").
timeout How long, in minutes, to keep trying to
find the image on the BMC. Default is 3 minutes.
"""
image_path = get_image_path(image_version)
image_version_id = image_path.split("/")[-2]
keyword.run_key_u("Open Connection And Log In")
image_purpose = get_image_purpose(image_path + "MANIFEST")
if (image_purpose == var.VERSION_PURPOSE_BMC
or image_purpose == var.VERSION_PURPOSE_HOST):
uri = var.SOFTWARE_VERSION_URI + image_version_id
ret_values = ""
for itr in range(timeout * 2):
status, ret_values = \
keyword.run_key("Read Attribute " + uri + " Activation")
if ((ret_values == var.READY) or (ret_values == var.INVALID)
or (ret_values == var.ACTIVE)):
return True, image_version_id
else:
time.sleep(30)
# If we exit the for loop, the timeout has been reached
gp.print_var(ret_values)
return False, None
else:
gp.print_var(image_purpose)
return False, None
def wait_for_activation_state_change(version_id, initial_state):
r"""
Wait for the current activation state of ${version_id} to
change from the state provided by the calling function.
Description of argument(s):
version_id The version ID whose state change we are
waiting for.
initial_state The activation state we want to wait for.
"""
keyword.run_key_u("Open Connection And Log In")
retry = 0
num_read_errors = 0
read_fail_threshold = 1
while (retry < 30):
# TODO: Use retry option in run_key when available.
status, software_state = keyword.run_key("Read Properties "
+ var.SOFTWARE_VERSION_URI
+ str(version_id),
ignore=1)
if status == 'FAIL':
num_read_errors += 1
if num_read_errors > read_fail_threshold:
message = "Read errors exceeds threshold:\n " \
+ gp.sprint_vars(num_read_errors, read_fail_threshold)
BuiltIn().fail(message)
time.sleep(10)
continue
current_state = (software_state)["Activation"]
if (initial_state == current_state):
time.sleep(10)
retry += 1
num_read_errors = 0
else:
return
return
def wait_for_activation_state_change(version_id, initial_state):
r"""
Wait for the current activation state of ${version_id} to
change from the state provided by the calling function.
Description of argument(s):
version_id The version ID whose state change we are
waiting for.
initial_state The activation state we want to wait for.
"""
keyword.run_key_u("Open Connection And Log In")
retry = 0
num_read_errors = 0
read_fail_threshold = 1
while (retry < 60):
# TODO: Use retry option in run_key when available.
status, software_state = keyword.run_key("Read Properties "
+ var.SOFTWARE_VERSION_URI
+ str(version_id),
ignore=1)
if status == 'FAIL':
num_read_errors += 1
if num_read_errors > read_fail_threshold:
message = "Read errors exceeds threshold:\n " \
+ gp.sprint_vars(num_read_errors, read_fail_threshold)
BuiltIn().fail(message)
time.sleep(10)
continue
current_state = (software_state)["Activation"]
if (initial_state == current_state):
time.sleep(10)
retry += 1
num_read_errors = 0
else:
return
return
def wait_for_activation_state_change(version_id, initial_state):
r"""
Wait for the current activation state of ${version_id} to
change from the state provided by the calling function.
Description of argument(s):
version_id The version ID whose state change we are waiting for.
initial_state The activation state we want to wait for.
"""
keyword.run_key_u("Open Connection And Log In")
retry = 0
while (retry < 20):
status, software_state = keyword.run_key("Read Properties " +
var.SOFTWARE_VERSION_URI +
str(version_id))
current_state = (software_state)["Activation"]
if (initial_state == current_state):
time.sleep(60)
retry += 1
else:
return
return
def test_loop_body():
r"""
The main loop body for the loop in main_py.
Description of arguments:
boot_count The iteration number (starts at 1).
"""
global boot_count
global state
global next_boot
global boot_success
gp.qprintn()
next_boot = select_boot()
if next_boot == "":
return True
boot_count += 1
gp.qprint_timen("Starting boot " + str(boot_count) + ".")
pre_boot_plug_in_setup()
cmd_buf = ["run_boot", next_boot]
boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
if boot_status == "FAIL":
gp.qprint(msg)
gp.qprintn()
if boot_status == "PASS":
boot_success = 1
gp.qprint_timen("BOOT_SUCCESS: \"" + next_boot + "\" succeeded.")
else:
boot_success = 0
gp.qprint_timen("BOOT_FAILED: \"" + next_boot + "\" failed.")
boot_results.update(next_boot, boot_status)
plug_in_setup()
# NOTE: A post_test_case call point failure is NOT counted as a boot
# failure.
rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
call_point='post_test_case', stop_on_plug_in_failure=0)
plug_in_setup()
rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
call_point='ffdc_check', shell_rc=0x00000200,
stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200:
status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
if status != 'PASS':
gp.print_error("Call to my_ffdc failed.\n")
# We need to purge error logs between boots or they build up.
grk.run_key("Delete Error logs", ignore=1)
boot_results.print_report()
gp.qprint_timen("Finished boot " + str(boot_count) + ".")
plug_in_setup()
rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
call_point='stop_check')
if rc != 0:
error_message = "Stopping as requested by user.\n"
grp.rprint_error_report(error_message)
BuiltIn().fail(error_message)
# This should help prevent ConnectionErrors.
grk.run_key_u("Close All Connections")
return True
def ffdc(ffdc_dir_path=None,
ffdc_prefix=None,
ffdc_function_list=""):
r"""
Gather First Failure Data Capture (FFDC).
This includes:
- Set global FFDC_TIME.
- Create FFDC work space directory.
- Write test info details.
- Call BMC methods to write/collect FFDC data.
Description of arguments:
ffdc_dir_path The dir path where FFDC data should be put.
ffdc_prefix The prefix to be given to each FFDC file name
generated.
ffdc_function_list A colon-delimited list of all the types of FFDC data
you wish to have collected. A blank value means that
all possible kinds of FFDC are to be collected. See
FFDC_METHOD_CALL object in lib/openbmc_ffdc_list.py
for possible choices.
"""
ffdc_file_list = []
# Check if Ping and SSH connection is alive
OPENBMC_HOST = BuiltIn().get_variable_value("${OPENBMC_HOST}")
state = st.get_state(req_states=['ping', 'uptime'])
gp.qprint_var(state)
if not int(state['ping']):
gp.print_error("BMC is not ping-able. Terminating FFDC collection.\n")
return ffdc_file_list
if state['uptime'] == "":
gp.print_error("BMC is not communicating. Terminating FFDC"
+ " collection.\n")
return ffdc_file_list
gp.qprint_timen("Collecting FFDC.")
# Get default values for arguments.
ffdc_dir_path, ffdc_prefix = set_ffdc_defaults(ffdc_dir_path, ffdc_prefix)
gp.qprint_var(ffdc_dir_path)
gp.qprint_var(ffdc_prefix)
# LOG_PREFIX is used by subordinate functions.
LOG_PREFIX = ffdc_dir_path + ffdc_prefix
BuiltIn().set_global_variable("${LOG_PREFIX}", LOG_PREFIX)
cmd_buf = ["Create Directory", ffdc_dir_path]
grp.rqpissuing_keyword(cmd_buf)
status, output = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
if status != "PASS":
error_message = grp.sprint_error_report("Create Directory failed"
+ " with the following"
+ " error:\n" + output)
BuiltIn().fail(error_message)
# FFDC_FILE_PATH is used by Header Message.
FFDC_FILE_PATH = ffdc_dir_path + ffdc_prefix + "BMC_general.txt"
BuiltIn().set_global_variable("${FFDC_FILE_PATH}", FFDC_FILE_PATH)
status, ffdc_file_list = grk.run_key("Header Message")
status, ffdc_file_sub_list = \
grk.run_key_u("Call FFDC Methods ffdc_function_list="
+ ffdc_function_list)
# Combine lists, remove duplicates and sort.
ffdc_file_list = sorted(set(ffdc_file_list + ffdc_file_sub_list))
gp.qprint_timen("Finished collecting FFDC.")
return ffdc_file_list
def setup():
r"""
Do general program setup tasks.
"""
global cp_setup_called
gp.qprintn()
robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
# If we can't find process_plug_in_packages.py, ssh_pw or
# validate_plug_ins.py, then we don't have our repo bin in PATH.
shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" +
" ssh_pw validate_plug_ins.py",
quiet=1,
print_output=0,
show_err=0)
if shell_rc != 0:
os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "")
# Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
if robot_pgm_dir_path not in sys.path:
sys.path.append(robot_pgm_dir_path)
PYTHONPATH = os.environ.get("PYTHONPATH", "")
if PYTHONPATH == "":
os.environ['PYTHONPATH'] = robot_pgm_dir_path
else:
os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH
validate_parms()
grp.rqprint_pgm_header()
grk.run_key("Set BMC Power Policy RESTORE_LAST_STATE")
initial_plug_in_setup()
plug_in_setup()
rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
call_point='setup')
if rc != 0:
error_message = "Plug-in setup failed.\n"
grp.rprint_error_report(error_message)
BuiltIn().fail(error_message)
# Setting cp_setup_called lets our Teardown know that it needs to call
# the cleanup plug-in call point.
cp_setup_called = 1
# Keyword "FFDC" will fail if TEST_MESSAGE is not set.
BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
# FFDC_LOG_PATH is used by "FFDC" keyword.
BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
# Also printed by FFDC.
global host_name
global host_ip
host = socket.gethostname()
host_name, host_ip = gm.get_host_name_ip(host)
gp.dprint_var(boot_table, 1)
gp.dprint_var(boot_lists)
def test_loop_body():
r"""
The main loop body for the loop in main_py.
Description of arguments:
boot_count The iteration number (starts at 1).
"""
global boot_count
global state
global next_boot
global boot_success
global boot_end_time
gp.qprintn()
next_boot = select_boot()
if next_boot == "":
return True
boot_count += 1
gp.qprint_timen("Starting boot " + str(boot_count) + ".")
pre_boot_plug_in_setup()
cmd_buf = ["run_boot", next_boot]
boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
if boot_status == "FAIL":
gp.qprint(msg)
gp.qprintn()
if boot_status == "PASS":
boot_success = 1
completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot
+ "\" succeeded.")
else:
boot_success = 0
completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot
+ "\" failed.")
# Set boot_end_time for use by plug-ins.
boot_end_time = completion_msg[1:33]
gp.qprint_var(boot_end_time)
gp.qprint(completion_msg)
boot_results.update(next_boot, boot_status)
plug_in_setup()
# NOTE: A post_test_case call point failure is NOT counted as a boot
# failure.
rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
call_point='post_test_case', stop_on_plug_in_failure=0)
plug_in_setup()
rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
call_point='ffdc_check', shell_rc=dump_ffdc_rc(),
stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
if ffdc_check == "All" or\
shell_rc == dump_ffdc_rc():
status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
if status != 'PASS':
gp.qprint_error("Call to my_ffdc failed.\n")
# Leave a record for caller that "soft" errors occurred.
soft_errors = 1
gpu.save_plug_in_value(soft_errors, pgm_name)
if delete_errlogs:
# print error logs before delete
status, error_logs = grk.run_key_u("Get Error Logs")
pels = pel.peltool("-l", ignore_err=1)
log.print_error_logs(error_logs, "AdditionalData Message Severity")
gp.qprint_var(pels)
# We need to purge error logs between boots or they build up.
grk.run_key(delete_errlogs_cmd, ignore=1)
grk.run_key(delete_bmcdump_cmd, ignore=1)
boot_results.print_report()
gp.qprint_timen("Finished boot " + str(boot_count) + ".")
plug_in_setup()
rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
call_point='stop_check', shell_rc=stop_test_rc(),
stop_on_non_zero_rc=1)
if shell_rc == stop_test_rc():
message = "Stopping as requested by user.\n"
gp.qprint_time(message)
BuiltIn().fail(message)
# This should help prevent ConnectionErrors.
# Purge all redfish and REST connection sessions.
if redfish_delete_sessions:
grk.run_key_u("Close All Connections", ignore=1)
grk.run_key_u("Delete All Redfish Sessions", ignore=1)
return True
def ffdc(ffdc_dir_path=None, ffdc_prefix=None, ffdc_function_list=""):
r"""
Gather First Failure Data Capture (FFDC).
This includes:
- Set global FFDC_TIME.
- Create FFDC work space directory.
- Write test info details.
- Call BMC methods to write/collect FFDC data.
Description of arguments:
ffdc_dir_path The dir path where FFDC data should be put.
ffdc_prefix The prefix to be given to each FFDC file name
generated.
ffdc_function_list A colon-delimited list of all the types of FFDC data
you wish to have collected. A blank value means that
all possible kinds of FFDC are to be collected. See
FFDC_METHOD_CALL object in lib/openbmc_ffdc_list.py
for possible choices.
"""
ffdc_file_list = []
# Check if Ping and SSH connection is alive
OPENBMC_HOST = BuiltIn().get_variable_value("${OPENBMC_HOST}")
state = st.get_state(req_states=['ping', 'uptime'])
gp.qprint_var(state)
if not int(state['ping']):
gp.print_error("BMC is not ping-able. Terminating FFDC collection.\n")
return ffdc_file_list
if state['uptime'] == "":
gp.print_error("BMC is not communicating. Terminating FFDC" +
" collection.\n")
return ffdc_file_list
gp.qprint_timen("Collecting FFDC.")
# Get default values for arguments.
ffdc_dir_path, ffdc_prefix = set_ffdc_defaults(ffdc_dir_path, ffdc_prefix)
gp.qprint_var(ffdc_dir_path)
gp.qprint_var(ffdc_prefix)
# LOG_PREFIX is used by subordinate functions.
LOG_PREFIX = ffdc_dir_path + ffdc_prefix
BuiltIn().set_global_variable("${LOG_PREFIX}", LOG_PREFIX)
cmd_buf = ["Create Directory", ffdc_dir_path]
grp.rqpissuing_keyword(cmd_buf)
status, output = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
if status != "PASS":
error_message = grp.sprint_error_report("Create Directory failed" +
" with the following" +
" error:\n" + output)
BuiltIn().fail(error_message)
# FFDC_FILE_PATH is used by Header Message.
FFDC_FILE_PATH = ffdc_dir_path + ffdc_prefix + "BMC_general.txt"
BuiltIn().set_global_variable("${FFDC_FILE_PATH}", FFDC_FILE_PATH)
status, ffdc_file_list = grk.run_key("Header Message")
status, ffdc_file_sub_list = \
grk.run_key_u("Call FFDC Methods ffdc_function_list="
+ ffdc_function_list)
# Combine lists, remove duplicates and sort.
ffdc_file_list = sorted(set(ffdc_file_list + ffdc_file_sub_list))
gp.qprint_timen("Finished collecting FFDC.")
return ffdc_file_list
def test_loop_body():
r"""
The main loop body for the loop in main_py.
Description of arguments:
boot_count The iteration number (starts at 1).
"""
global boot_count
global state
global next_boot
global boot_success
global boot_end_time
gp.qprintn()
next_boot = select_boot()
if next_boot == "":
return True
boot_count += 1
gp.qprint_timen("Starting boot " + str(boot_count) + ".")
pre_boot_plug_in_setup()
cmd_buf = ["run_boot", next_boot]
boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
if boot_status == "FAIL":
gp.qprint(msg)
gp.qprintn()
if boot_status == "PASS":
boot_success = 1
completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot +
"\" succeeded.")
else:
boot_success = 0
completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot +
"\" failed.")
# Set boot_end_time for use by plug-ins.
boot_end_time = completion_msg[1:33]
gp.qprint_var(boot_end_time)
gp.qprint(completion_msg)
boot_results.update(next_boot, boot_status)
plug_in_setup()
# NOTE: A post_test_case call point failure is NOT counted as a boot
# failure.
rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
call_point='post_test_case', stop_on_plug_in_failure=0)
plug_in_setup()
rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
call_point='ffdc_check',
shell_rc=0x00000200,
stop_on_plug_in_failure=1,
stop_on_non_zero_rc=1)
if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200:
status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
if status != 'PASS':
gp.qprint_error("Call to my_ffdc failed.\n")
# We need to purge error logs between boots or they build up.
grk.run_key("Delete Error logs", ignore=1)
boot_results.print_report()
gp.qprint_timen("Finished boot " + str(boot_count) + ".")
plug_in_setup()
rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
call_point='stop_check')
if rc != 0:
error_message = "Stopping as requested by user.\n"
grp.rprint_error_report(error_message)
BuiltIn().fail(error_message)
# This should help prevent ConnectionErrors.
grk.run_key_u("Close All Connections")
return True
def obmc_boot_test_py(loc_boot_stack=None,
loc_stack_mode=None,
loc_quiet=None):
r"""
Do main program processing.
"""
global save_stack
ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']})
gp.dprintn()
# Process function parms.
for parm_name in main_func_parm_list:
# Get parm's value.
parm_value = eval("loc_" + parm_name)
gp.dpvars(parm_name, parm_value)
if parm_value is not None:
# Save the global value on a stack.
cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\
parm_name + "}\"), \"" + parm_name + "\")"
gp.dpissuing(cmd_buf)
exec(cmd_buf)
# Set the global value to the passed value.
cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
"}\", loc_" + parm_name + ")"
gp.dpissuing(cmd_buf)
exec(cmd_buf)
gp.dprintn(save_stack.sprint_obj())
setup()
init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail()
if ffdc_only:
gp.qprint_timen("Caller requested ffdc_only.")
if do_pre_boot_plug_in_setup:
pre_boot_plug_in_setup()
grk.run_key_u("my_ffdc")
return
if delete_errlogs:
# print error logs before delete
status, error_logs = grk.run_key_u("Get Error Logs")
pels = pel.peltool("-l", ignore_err=1)
log.print_error_logs(error_logs, "AdditionalData Message Severity")
gp.qprint_var(pels)
# Delete errlogs prior to doing any boot tests.
grk.run_key(delete_errlogs_cmd, ignore=1)
grk.run_key(delete_bmcdump_cmd, ignore=1)
# Process caller's boot_stack.
while (len(boot_stack) > 0):
test_loop_body()
gp.qprint_timen("Finished processing stack.")
post_stack()
# Process caller's boot_list.
if len(boot_list) > 0:
for ix in range(1, max_num_tests + 1):
test_loop_body()
gp.qprint_timen("Completed all requested boot tests.")
boot_pass, boot_fail = boot_results.return_total_pass_fail()
new_fail = boot_fail - init_boot_fail
if new_fail > boot_fail_threshold:
error_message = "Boot failures exceed the boot failure" +\
" threshold:\n" +\
gp.sprint_var(new_fail) +\
gp.sprint_var(boot_fail_threshold)
BuiltIn().fail(gp.sprint_error(error_message))
