def stop_boot_test(signal_number=0, frame=None): r""" Handle SIGUSR1 by aborting the boot test that is running. Description of argument(s): signal_number The signal number (should always be 10 for SIGUSR1). frame The frame data. """ gp.qprintn() gp.qprint_executing() gp.lprint_executing() # Restore original sigusr1 handler. set_default_siguser1() message = "The caller has asked that the boot test be stopped and marked" message += " as a failure." function_stack = gm.get_function_stack() if "wait_state" in function_stack: st.set_exit_wait_early_message(message) else: BuiltIn().fail(gp.sprint_error(message))
def test_setup(): r""" Do test case setup tasks. """ gp.qprintn() gp.qprint_executing()
def setup(): r""" Do general program setup tasks. """ global cp_setup_called gp.qprintn() robot_pgm_dir_path = os.path.dirname(__file__) + os.sep repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") # If we can't find process_plug_in_packages.py, ssh_pw or # validate_plug_ins.py, then we don't have our repo bin in PATH. shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" + " ssh_pw validate_plug_ins.py", quiet=1, print_output=0, show_err=0) if shell_rc != 0: os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. if robot_pgm_dir_path not in sys.path: sys.path.append(robot_pgm_dir_path) PYTHONPATH = os.environ.get("PYTHONPATH", "") if PYTHONPATH == "": os.environ['PYTHONPATH'] = robot_pgm_dir_path else: os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH validate_parms() grp.rqprint_pgm_header() grk.run_key("Set BMC Power Policy RESTORE_LAST_STATE") initial_plug_in_setup() plug_in_setup() rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( call_point='setup') if rc != 0: error_message = "Plug-in setup failed.\n" grp.rprint_error_report(error_message) BuiltIn().fail(error_message) # Setting cp_setup_called lets our Teardown know that it needs to call # the cleanup plug-in call point. cp_setup_called = 1 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") # FFDC_LOG_PATH is used by "FFDC" keyword. BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) # Also printed by FFDC. global host_name global host_ip host = socket.gethostname() host_name, host_ip = gm.get_host_name_ip(host) gp.dprint_var(boot_table, 1) gp.dprint_var(boot_lists)
def validate_parms(): r""" Validate all program parameters. """ process_pgm_parms() gp.qprintn() global openbmc_model grv.rvalid_value("openbmc_host") grv.rvalid_value("openbmc_username") grv.rvalid_value("openbmc_password") if os_host != "": grv.rvalid_value("os_username") grv.rvalid_value("os_password") if pdu_host != "": grv.rvalid_value("pdu_username") grv.rvalid_value("pdu_password") grv.rvalid_integer("pdu_slot_no") if openbmc_serial_host != "": grv.rvalid_integer("openbmc_serial_port") if openbmc_model == "": status, ret_values =\ grk.run_key_u("Get BMC System Model") openbmc_model = ret_values BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) grv.rvalid_value("openbmc_model") grv.rvalid_integer("max_num_tests") grv.rvalid_integer("boot_pass") grv.rvalid_integer("boot_fail") plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) BuiltIn().set_global_variable("${plug_in_packages_list}", plug_in_packages_list) grv.rvalid_value("stack_mode", valid_values=['normal', 'skip']) if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: error_message = "You must provide either a value for either the" +\ " boot_list or the boot_stack parm.\n" BuiltIn().fail(gp.sprint_error(error_message)) valid_boot_list(boot_list, valid_boot_types) valid_boot_list(boot_stack, valid_boot_types) selected_PDU_boots = list(set(boot_list + boot_stack) & set(boot_lists['PDU_reboot'])) if len(selected_PDU_boots) > 0 and pdu_host == "": error_message = "You have selected the following boots which" +\ " require a PDU host but no value for pdu_host:\n" error_message += gp.sprint_var(selected_PDU_boots) error_message += gp.sprint_var(pdu_host, 2) BuiltIn().fail(gp.sprint_error(error_message)) return
def validate_parms(): r""" Validate all program parameters. """ process_pgm_parms() gp.qprintn() global openbmc_model grv.rvalid_value("openbmc_host") grv.rvalid_value("openbmc_username") grv.rvalid_value("openbmc_password") if os_host != "": grv.rvalid_value("os_username") grv.rvalid_value("os_password") if pdu_host != "": grv.rvalid_value("pdu_username") grv.rvalid_value("pdu_password") grv.rvalid_integer("pdu_slot_no") if openbmc_serial_host != "": grv.rvalid_integer("openbmc_serial_port") if openbmc_model == "": status, ret_values =\ grk.run_key_u("Get BMC System Model") openbmc_model = ret_values BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) grv.rvalid_value("openbmc_model") grv.rvalid_integer("max_num_tests") grv.rvalid_integer("boot_pass") grv.rvalid_integer("boot_fail") plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) BuiltIn().set_global_variable("${plug_in_packages_list}", plug_in_packages_list) grv.rvalid_value("stack_mode", valid_values=['normal', 'skip']) if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: error_message = "You must provide either a value for either the" +\ " boot_list or the boot_stack parm.\n" BuiltIn().fail(gp.sprint_error(error_message)) valid_boot_list(boot_list, valid_boot_types) valid_boot_list(boot_stack, valid_boot_types) selected_PDU_boots = list( set(boot_list + boot_stack) & set(boot_lists['PDU_reboot'])) if len(selected_PDU_boots) > 0 and pdu_host == "": error_message = "You have selected the following boots which" +\ " require a PDU host but no value for pdu_host:\n" error_message += gp.sprint_var(selected_PDU_boots) error_message += gp.sprint_var(pdu_host, 2) BuiltIn().fail(gp.sprint_error(error_message)) return
def set_default_siguser1(): r""" Set the default_sigusr1 function to be the SIGUSR1 handler. """ gp.qprintn() gp.qprint_executing() gp.lprint_executing() signal.signal(signal.SIGUSR1, default_sigusr1)
def suite_setup(): r""" Do test suite setup tasks. """ gp.qprintn() validate_suite_parms() gp.qprint_pgm_header()
def setup(): r""" Do general program setup tasks. """ gp.qprintn() validate_parms() gp.qprint_pgm_header()
def test_teardown(): r""" Clean up after this test case. """ gp.qprintn() cmd_buf = ["Print Error", "A keyword timeout occurred ending this program.\n"] BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) grp.rqprint_pgm_footer()
def print_last_boots(): r""" Print the last ten boots done with their time stamps. """ # indent 0, 90 chars wide, linefeed, char is "=" gp.qprint_dashes(0, 90) gp.qprintn("Last 10 boots:\n") for boot_entry in last_ten: grp.rqprint(boot_entry) gp.qprint_dashes(0, 90)
def test_teardown(): r""" Clean up after this test case. """ gp.qprintn() cmd_buf = [ "Print Error", "A keyword timeout occurred ending this program.\n" ] BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) grp.rqprint_pgm_footer()
def default_sigusr1(signal_number=0, frame=None): r""" Handle SIGUSR1 by doing nothing. This function assists in debugging SIGUSR1 processing by printing messages to stdout and to the log.html file. Description of argument(s): signal_number The signal number (should always be 10 for SIGUSR1). frame The frame data. """ gp.qprintn() gp.qprint_executing() gp.lprint_executing()
def rmtree(path, ignore_errors=False, onerror=None, quiet=None): r""" Call shutil.rmtree with the caller's arguments. This function offers this advantage over the base function: - It will print an "Issuing: shutil.rmtree" message. Description of argument(s): (All parms are passed directly to shutil.rmtree. See its prolog for details) quiet Indicates whether this function should run the print_issuing() function. """ quiet = int(dft(quiet, gp.get_stack_var('quiet', 0))) print_string = gp.sprint_executing(max_width=2000) print_string = re.sub(r"Executing: ", "Issuing: shutil.", print_string.rstrip("\n")) gp.qprintn(re.sub(r", quiet[ ]?=.*", ")", print_string)) shutil.rmtree(path, ignore_errors, onerror)
def test_teardown(): r""" Clean up after this test case. """ gp.qprintn() gp.qprint_executing() if ga.psutil_imported: ga.terminate_descendants() cmd_buf = ["Print Error", "A keyword timeout occurred ending this program.\n"] BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) gp.qprint_pgm_footer()
def print_boot_history(boot_history, quiet=None): r""" Print the last ten boots done with their time stamps. Description of argument(s): quiet Only print if this value is 0. This function will search upward in the stack to get the default value. """ quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0))) # indent 0, 90 chars wide, linefeed, char is "=" gp.qprint_dashes(0, 90) gp.qprintn("Last 10 boots:\n") for boot_entry in boot_history: gp.qprint(boot_entry) gp.qprint_dashes(0, 90)
def wait_for_reboot(start_boot_seconds): r""" Wait for the BMC to complete a previously initiated reboot. Description of argument(s): start_boot_seconds The time that the boot test started. The format is the epoch time in seconds, i.e. the number of seconds since 1970-01-01 00:00:00 UTC. This value should be obtained from the BMC so that it is not dependent on any kind of synchronization between this machine and the target BMC This will allow this program to work correctly even in a simulated environment. This value should be obtained by the caller prior to initiating a reboot. It can be obtained as follows: state = st.get_state(req_states=['epoch_seconds']) """ st.wait_for_comm_cycle(int(start_boot_seconds)) gp.qprintn() st.wait_state(st.standby_match_state, wait_time="10 mins", interval="10 seconds")
def run_boot(boot): r""" Run the specified boot. Description of arguments: boot The name of the boot test to be performed. """ global state print_test_start_message(boot) plug_in_setup() rc, shell_rc, failed_plug_in_name = \ grpi.rprocess_plug_in_packages(call_point="pre_boot") if rc != 0: error_message = "Plug-in failed with non-zero return code.\n" +\ gp.sprint_var(rc, 1) BuiltIn().fail(gp.sprint_error(error_message)) if test_mode: # In test mode, we'll pretend the boot worked by assigning its # required end state to the default state value. state = st.strip_anchor_state(boot_table[boot]['end']) else: # Assertion: We trust that the state data was made fresh by the # caller. gp.qprintn() if boot_table[boot]['method_type'] == "keyword": rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), boot_table[boot]['method'], quiet=quiet) if boot_table[boot]['bmc_reboot']: st.wait_for_comm_cycle(int(state['epoch_seconds'])) plug_in_setup() rc, shell_rc, failed_plug_in_name = \ grpi.rprocess_plug_in_packages(call_point="post_reboot") if rc != 0: error_message = "Plug-in failed with non-zero return code.\n" error_message += gp.sprint_var(rc, 1) BuiltIn().fail(gp.sprint_error(error_message)) else: match_state = st.anchor_state(state) del match_state['epoch_seconds'] # Wait for the state to change in any way. st.wait_state(match_state, wait_time=state_change_timeout, interval="10 seconds", invert=1) gp.qprintn() if boot_table[boot]['end']['chassis'] == "Off": boot_timeout = power_off_timeout else: boot_timeout = power_on_timeout st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, interval="10 seconds") plug_in_setup() rc, shell_rc, failed_plug_in_name = \ grpi.rprocess_plug_in_packages(call_point="post_boot") if rc != 0: error_message = "Plug-in failed with non-zero return code.\n" +\ gp.sprint_var(rc, 1) BuiltIn().fail(gp.sprint_error(error_message))
def test_loop_body(): r""" The main loop body for the loop in main_py. Description of arguments: boot_count The iteration number (starts at 1). """ global boot_count global state global next_boot global boot_success global boot_end_time gp.qprintn() next_boot = select_boot() if next_boot == "": return True boot_count += 1 gp.qprint_timen("Starting boot " + str(boot_count) + ".") pre_boot_plug_in_setup() cmd_buf = ["run_boot", next_boot] boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) if boot_status == "FAIL": gp.qprint(msg) gp.qprintn() if boot_status == "PASS": boot_success = 1 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot + "\" succeeded.") else: boot_success = 0 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot + "\" failed.") # Set boot_end_time for use by plug-ins. boot_end_time = completion_msg[1:33] gp.qprint_var(boot_end_time) gp.qprint(completion_msg) boot_results.update(next_boot, boot_status) plug_in_setup() # NOTE: A post_test_case call point failure is NOT counted as a boot # failure. rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( call_point='post_test_case', stop_on_plug_in_failure=0) plug_in_setup() rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( call_point='ffdc_check', shell_rc=0x00000200, stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200: status, ret_values = grk.run_key_u("my_ffdc", ignore=1) if status != 'PASS': gp.qprint_error("Call to my_ffdc failed.\n") # We need to purge error logs between boots or they build up. grk.run_key("Delete Error logs", ignore=1) boot_results.print_report() gp.qprint_timen("Finished boot " + str(boot_count) + ".") plug_in_setup() rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( call_point='stop_check') if rc != 0: error_message = "Stopping as requested by user.\n" grp.rprint_error_report(error_message) BuiltIn().fail(error_message) # This should help prevent ConnectionErrors. grk.run_key_u("Close All Connections") return True
def print_defect_report(): r""" Print a defect report. """ # Making deliberate choice to NOT run plug_in_setup(). We don't want # ffdc_prefix updated. rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( call_point='ffdc_report', stop_on_plug_in_failure=0) # At some point I'd like to have the 'Call FFDC Methods' return a list # of files it has collected. In that case, the following "ls" command # would no longer be needed. For now, however, glob shows the files # named in FFDC_LIST_FILE_PATH so I will refrain from printing those # out (so we don't see duplicates in the list). # Get additional header data which may have been created by ffdc plug-ins. # Also, delete the individual header files to cleanup. cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, show_err=0) # Get additional header data which may have been created by ffdc plug-ins. # Also, delete the individual header files to cleanup. cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, show_err=0) LOG_PREFIX = BuiltIn().get_variable_value("${LOG_PREFIX}") output = '\n'.join(sorted(glob.glob(LOG_PREFIX + '*'))) try: ffdc_list = open(ffdc_list_file_path, 'r') except IOError: ffdc_list = "" # Open ffdc_file_list for writing. We will write a complete list of # FFDC files to it for possible use by plug-ins like cp_stop_check. ffdc_list_file = open(ffdc_list_file_path, 'w') gp.qprintn() # indent=0, width=90, linefeed=1, char="=" gp.qprint_dashes(0, 90, 1, "=") gp.qprintn("Copy this data to the defect:\n") if len(more_header_info) > 0: gp.qprintn(more_header_info) gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, openbmc_host_name, openbmc_ip, openbmc_username, openbmc_password, os_host, os_host_name, os_ip, os_username, os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, pdu_password, pdu_slot_no, openbmc_serial_host, openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) gp.qprintn() print_last_boots() gp.qprintn() gp.qprint_var(state) gp.qprintn() gp.qprintn("FFDC data files:") if status_file_path != "": gp.qprintn(status_file_path) ffdc_list_file.write(status_file_path + "\n") gp.qprintn(output) # gp.qprintn(ffdc_list) gp.qprintn() if len(ffdc_summary_info) > 0: gp.qprintn(ffdc_summary_info) gp.qprint_dashes(0, 90, 1, "=") ffdc_list_file.write(output + "\n") ffdc_list_file.close()
def process_robot_output_files(robot_cmd_buf=None, robot_rc=None, gzip=1): r""" Process robot output files which can involve several operations: - If the files are in a temporary location, using SAVE_STATUS_POLICY to decide whether to move them to a permanent location or to delete them. - Gzipping them. Description of argument(s): robot_cmd_buf The complete command string used to invoke robot. robot_rc The return code from running the robot command string. gzip Indicates whether robot-generated output should be gzipped. """ robot_cmd_buf = gm.dft(robot_cmd_buf, gcr_last_robot_cmd_buf) robot_rc = gm.dft(robot_rc, gcr_last_robot_rc) if robot_cmd_buf == "": # This can legitimately occur if this function is called from an # exit_function without the program having ever run robot_cmd_fnc. return SAVE_STATUS_POLICY = os.environ.get("SAVE_STATUS_POLICY", "ALWAYS") gp.qprint_vars(SAVE_STATUS_POLICY) # When SAVE_STATUS_POLICY is "NEVER" robot output files don't even get # generated. if SAVE_STATUS_POLICY == "NEVER": return # Compose file_list based on robot command buffer passed in. robot_cmd_buf_dict = gc.parse_command_string(robot_cmd_buf) outputdir = robot_cmd_buf_dict['outputdir'] outputdir = gm.add_trailing_slash(outputdir) file_list = outputdir + robot_cmd_buf_dict['output'] + " " + outputdir\ + robot_cmd_buf_dict['log'] + " " + outputdir\ + robot_cmd_buf_dict['report'] # Double checking that files are present. shell_rc, out_buf = gc.shell_cmd("ls -1 " + file_list + " 2>/dev/null", show_err=0) file_list = re.sub("\n", " ", out_buf.rstrip("\n")) if file_list == "": gp.qprint_timen("No robot output files were found in " + outputdir + ".") return gp.qprint_var(robot_rc, 1) if SAVE_STATUS_POLICY == "FAIL" and robot_rc == 0: gp.qprint_timen("The call to robot produced no failures." + " Deleting robot output files.") gc.shell_cmd("rm -rf " + file_list) return if gzip: gc.shell_cmd("gzip " + file_list) # Update the values in file_list. file_list = re.sub(" ", ".gz ", file_list) + ".gz" # It TMP_ROBOT_DIR_PATH is set, it means the caller wanted the robot # output initially directed to TMP_ROBOT_DIR_PATH but later moved to # FFDC_DIR_PATH. Otherwise, we're done. if os.environ.get("TMP_ROBOT_DIR_PATH", "") is "": return # We're directing these to the FFDC dir path so that they'll be subjected # to FFDC cleanup. target_dir_path = os.environ.get("FFDC_DIR_PATH", os.environ.get("HOME", ".") + "/autoipl/ffdc") target_dir_path = gm.add_trailing_slash(target_dir_path) targ_file_list = [re.sub(".*/", target_dir_path, x) for x in file_list.split(" ")] gc.shell_cmd("mv " + file_list + " " + target_dir_path + " >/dev/null", time_out=600) gp.qprint_timen("New robot log file locations:") gp.qprintn('\n'.join(targ_file_list))
def select_boot(): r""" Select a boot test to be run based on our current state and return the chosen boot type. Description of arguments: state The state of the machine. """ global boot_stack gp.qprint_timen("Selecting a boot test.") my_get_state() stack_popped = 0 if len(boot_stack) > 0: stack_popped = 1 gp.qprint_dashes() gp.qprint_var(boot_stack) gp.qprint_dashes() skip_boot_printed = 0 while len(boot_stack) > 0: boot_candidate = boot_stack.pop() if stack_mode == 'normal': break else: if st.compare_states(state, boot_table[boot_candidate]['end']): if not skip_boot_printed: gp.qprint_var(stack_mode) gp.qprintn() gp.qprint_timen("Skipping the following boot tests" + " which are unnecessary since their" + " required end states match the" + " current machine state:") skip_boot_printed = 1 gp.qprint_var(boot_candidate) boot_candidate = "" if boot_candidate == "": gp.qprint_dashes() gp.qprint_var(boot_stack) gp.qprint_dashes() return boot_candidate if st.compare_states(state, boot_table[boot_candidate]['start']): gp.qprint_timen("The machine state is valid for a '" + boot_candidate + "' boot test.") gp.qprint_dashes() gp.qprint_var(boot_stack) gp.qprint_dashes() return boot_candidate else: gp.qprint_timen("The machine state does not match the required" + " starting state for a '" + boot_candidate + "' boot test:") gp.qprint_varx("boot_table[" + boot_candidate + "][start]", boot_table[boot_candidate]['start'], 1) boot_stack.append(boot_candidate) popped_boot = boot_candidate # Loop through your list selecting a boot_candidates boot_candidates = [] for boot_candidate in boot_list: if st.compare_states(state, boot_table[boot_candidate]['start']): if stack_popped: if st.compare_states(boot_table[boot_candidate]['end'], boot_table[popped_boot]['start']): boot_candidates.append(boot_candidate) else: boot_candidates.append(boot_candidate) if len(boot_candidates) == 0: gp.qprint_timen("The user's boot list contained no boot tests" + " which are valid for the current machine state.") boot_candidate = default_power_on if not st.compare_states(state, boot_table[default_power_on]['start']): boot_candidate = default_power_off boot_candidates.append(boot_candidate) gp.qprint_timen("Using default '" + boot_candidate + "' boot type to transition to valid state.") gp.dprint_var(boot_candidates) # Randomly select a boot from the candidate list. boot = random.choice(boot_candidates) return boot
def print_defect_report(ffdc_file_list): r""" Print a defect report. Description of argument(s): ffdc_file_list A list of files which were collected by our ffdc functions. """ # Making deliberate choice to NOT run plug_in_setup(). We don't want # ffdc_prefix updated. rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( call_point='ffdc_report', stop_on_plug_in_failure=0) # Get additional header data which may have been created by ffdc plug-ins. # Also, delete the individual header files to cleanup. cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, show_err=0) # Get additional summary data which may have been created by ffdc plug-ins. # Also, delete the individual header files to cleanup. cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, show_err=0) # ffdc_list_file_path contains a list of any ffdc files created by plug- # ins, etc. Read that data into a list. try: plug_in_ffdc_list = \ open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") plug_in_ffdc_list = filter(None, plug_in_ffdc_list) except IOError: plug_in_ffdc_list = [] # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed # in. Eliminate duplicates and sort the list. ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) if status_file_path != "": ffdc_file_list.insert(0, status_file_path) # Convert the list to a printable list. printable_ffdc_file_list = "\n".join(ffdc_file_list) # Open ffdc_file_list for writing. We will write a complete list of # FFDC files to it for possible use by plug-ins like cp_stop_check. ffdc_list_file = open(ffdc_list_file_path, 'w') ffdc_list_file.write(printable_ffdc_file_list + "\n") ffdc_list_file.close() indent = 0 width = 90 linefeed = 1 char = "=" gp.qprintn() gp.qprint_dashes(indent, width, linefeed, char) gp.qprintn("Copy this data to the defect:\n") if len(more_header_info) > 0: gp.qprintn(more_header_info) gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, openbmc_host_name, openbmc_ip, openbmc_username, openbmc_password, os_host, os_host_name, os_ip, os_username, os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, pdu_password, pdu_slot_no, openbmc_serial_host, openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) gp.qprintn() print_last_boots() gp.qprintn() gp.qprint_var(state) gp.qprintn() gp.qprintn("FFDC data files:") gp.qprintn(printable_ffdc_file_list) gp.qprintn() if len(ffdc_summary_info) > 0: gp.qprintn(ffdc_summary_info) gp.qprint_dashes(indent, width, linefeed, char)
def select_boot(): r""" Select a boot test to be run based on our current state and return the chosen boot type. Description of arguments: state The state of the machine. """ global transitional_boot_selected global boot_stack gp.qprint_timen("Selecting a boot test.") if transitional_boot_selected and not boot_success: prior_boot = next_boot boot_candidate = boot_stack.pop() gp.qprint_timen("The prior '" + next_boot + "' was chosen to" + " transition to a valid state for '" + boot_candidate + "' which was at the top of the boot_stack. Since" + " the '" + next_boot + "' failed, the '" + boot_candidate + "' has been removed from the stack" + " to avoid and endless failure loop.") if len(boot_stack) == 0: return "" my_get_state() valid_state() transitional_boot_selected = False stack_popped = 0 if len(boot_stack) > 0: stack_popped = 1 gp.qprint_dashes() gp.qprint_var(boot_stack) gp.qprint_dashes() skip_boot_printed = 0 while len(boot_stack) > 0: boot_candidate = boot_stack.pop() if stack_mode == 'normal': break else: if st.compare_states(state, boot_table[boot_candidate]['end']): if not skip_boot_printed: gp.qprint_var(stack_mode) gp.qprintn() gp.qprint_timen("Skipping the following boot tests" + " which are unnecessary since their" + " required end states match the" + " current machine state:") skip_boot_printed = 1 gp.qprint_var(boot_candidate) boot_candidate = "" if boot_candidate == "": gp.qprint_dashes() gp.qprint_var(boot_stack) gp.qprint_dashes() return boot_candidate if st.compare_states(state, boot_table[boot_candidate]['start']): gp.qprint_timen("The machine state is valid for a '" + boot_candidate + "' boot test.") gp.qprint_dashes() gp.qprint_var(boot_stack) gp.qprint_dashes() return boot_candidate else: gp.qprint_timen("The machine state does not match the required" + " starting state for a '" + boot_candidate + "' boot test:") gp.qprint_varx("boot_table_start_entry", boot_table[boot_candidate]['start']) boot_stack.append(boot_candidate) transitional_boot_selected = True popped_boot = boot_candidate # Loop through your list selecting a boot_candidates boot_candidates = [] for boot_candidate in boot_list: if st.compare_states(state, boot_table[boot_candidate]['start']): if stack_popped: if st.compare_states(boot_table[boot_candidate]['end'], boot_table[popped_boot]['start']): boot_candidates.append(boot_candidate) else: boot_candidates.append(boot_candidate) if len(boot_candidates) == 0: gp.qprint_timen("The user's boot list contained no boot tests" + " which are valid for the current machine state.") boot_candidate = default_power_on if not st.compare_states(state, boot_table[default_power_on]['start']): boot_candidate = default_power_off boot_candidates.append(boot_candidate) gp.qprint_timen("Using default '" + boot_candidate + "' boot type to transition to valid state.") gp.dprint_var(boot_candidates) # Randomly select a boot from the candidate list. boot = random.choice(boot_candidates) return boot
def test_loop_body(): r""" The main loop body for the loop in main_py. Description of arguments: boot_count The iteration number (starts at 1). """ global boot_count global state global next_boot global boot_success gp.qprintn() next_boot = select_boot() if next_boot == "": return True boot_count += 1 gp.qprint_timen("Starting boot " + str(boot_count) + ".") pre_boot_plug_in_setup() cmd_buf = ["run_boot", next_boot] boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) if boot_status == "FAIL": gp.qprint(msg) gp.qprintn() if boot_status == "PASS": boot_success = 1 gp.qprint_timen("BOOT_SUCCESS: \"" + next_boot + "\" succeeded.") else: boot_success = 0 gp.qprint_timen("BOOT_FAILED: \"" + next_boot + "\" failed.") boot_results.update(next_boot, boot_status) plug_in_setup() # NOTE: A post_test_case call point failure is NOT counted as a boot # failure. rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( call_point='post_test_case', stop_on_plug_in_failure=0) plug_in_setup() rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( call_point='ffdc_check', shell_rc=0x00000200, stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200: status, ret_values = grk.run_key_u("my_ffdc", ignore=1) if status != 'PASS': gp.print_error("Call to my_ffdc failed.\n") # We need to purge error logs between boots or they build up. grk.run_key("Delete Error logs", ignore=1) boot_results.print_report() gp.qprint_timen("Finished boot " + str(boot_count) + ".") plug_in_setup() rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( call_point='stop_check') if rc != 0: error_message = "Stopping as requested by user.\n" grp.rprint_error_report(error_message) BuiltIn().fail(error_message) # This should help prevent ConnectionErrors. grk.run_key_u("Close All Connections") return True
def test_loop_body(): r""" The main loop body for the loop in main_py. Description of arguments: boot_count The iteration number (starts at 1). """ global boot_count global state global next_boot global boot_success global boot_end_time gp.qprintn() next_boot = select_boot() if next_boot == "": return True boot_count += 1 gp.qprint_timen("Starting boot " + str(boot_count) + ".") pre_boot_plug_in_setup() cmd_buf = ["run_boot", next_boot] boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) if boot_status == "FAIL": gp.qprint(msg) gp.qprintn() if boot_status == "PASS": boot_success = 1 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot + "\" succeeded.") else: boot_success = 0 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot + "\" failed.") # Set boot_end_time for use by plug-ins. boot_end_time = completion_msg[1:33] gp.qprint_var(boot_end_time) gp.qprint(completion_msg) boot_results.update(next_boot, boot_status) plug_in_setup() # NOTE: A post_test_case call point failure is NOT counted as a boot # failure. rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( call_point='post_test_case', stop_on_plug_in_failure=0) plug_in_setup() rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( call_point='ffdc_check', shell_rc=dump_ffdc_rc(), stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) if ffdc_check == "All" or\ shell_rc == dump_ffdc_rc(): status, ret_values = grk.run_key_u("my_ffdc", ignore=1) if status != 'PASS': gp.qprint_error("Call to my_ffdc failed.\n") # Leave a record for caller that "soft" errors occurred. soft_errors = 1 gpu.save_plug_in_value(soft_errors, pgm_name) if delete_errlogs: # print error logs before delete status, error_logs = grk.run_key_u("Get Error Logs") pels = pel.peltool("-l", ignore_err=1) log.print_error_logs(error_logs, "AdditionalData Message Severity") gp.qprint_var(pels) # We need to purge error logs between boots or they build up. grk.run_key(delete_errlogs_cmd, ignore=1) grk.run_key(delete_bmcdump_cmd, ignore=1) boot_results.print_report() gp.qprint_timen("Finished boot " + str(boot_count) + ".") plug_in_setup() rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( call_point='stop_check', shell_rc=stop_test_rc(), stop_on_non_zero_rc=1) if shell_rc == stop_test_rc(): message = "Stopping as requested by user.\n" gp.qprint_time(message) BuiltIn().fail(message) # This should help prevent ConnectionErrors. # Purge all redfish and REST connection sessions. if redfish_delete_sessions: grk.run_key_u("Close All Connections", ignore=1) grk.run_key_u("Delete All Redfish Sessions", ignore=1) return True
def select_boot(): r""" Select a boot test to be run based on our current state and return the chosen boot type. Description of arguments: state The state of the machine. """ global transitional_boot_selected global boot_stack gp.qprint_timen("Selecting a boot test.") if transitional_boot_selected and not boot_success: prior_boot = next_boot boot_candidate = boot_stack.pop() gp.qprint_timen("The prior '" + next_boot + "' was chosen to" + " transition to a valid state for '" + boot_candidate + "' which was at the top of the boot_stack. Since" + " the '" + next_boot + "' failed, the '" + boot_candidate + "' has been removed from the stack" + " to avoid and endless failure loop.") if len(boot_stack) == 0: return "" my_get_state() valid_state() transitional_boot_selected = False stack_popped = 0 if len(boot_stack) > 0: stack_popped = 1 gp.qprint_dashes() gp.qprint_var(boot_stack) gp.qprint_dashes() skip_boot_printed = 0 while len(boot_stack) > 0: boot_candidate = boot_stack.pop() if stack_mode == 'normal': break else: if st.compare_states(state, boot_table[boot_candidate]['end']): if not skip_boot_printed: gp.qprint_var(stack_mode) gp.qprintn() gp.qprint_timen("Skipping the following boot tests" + " which are unnecessary since their" + " required end states match the" + " current machine state:") skip_boot_printed = 1 gp.qprint_var(boot_candidate) boot_candidate = "" if boot_candidate == "": gp.qprint_dashes() gp.qprint_var(boot_stack) gp.qprint_dashes() return boot_candidate if st.compare_states(state, boot_table[boot_candidate]['start']): gp.qprint_timen("The machine state is valid for a '" + boot_candidate + "' boot test.") gp.qprint_dashes() gp.qprint_var(boot_stack) gp.qprint_dashes() return boot_candidate else: gp.qprint_timen("The machine state does not match the required" + " starting state for a '" + boot_candidate + "' boot test:") gp.qprint_varx("boot_table[" + boot_candidate + "][start]", boot_table[boot_candidate]['start'], 1) boot_stack.append(boot_candidate) transitional_boot_selected = True popped_boot = boot_candidate # Loop through your list selecting a boot_candidates boot_candidates = [] for boot_candidate in boot_list: if st.compare_states(state, boot_table[boot_candidate]['start']): if stack_popped: if st.compare_states(boot_table[boot_candidate]['end'], boot_table[popped_boot]['start']): boot_candidates.append(boot_candidate) else: boot_candidates.append(boot_candidate) if len(boot_candidates) == 0: gp.qprint_timen("The user's boot list contained no boot tests" + " which are valid for the current machine state.") boot_candidate = default_power_on if not st.compare_states(state, boot_table[default_power_on]['start']): boot_candidate = default_power_off boot_candidates.append(boot_candidate) gp.qprint_timen("Using default '" + boot_candidate + "' boot type to transition to valid state.") gp.dprint_var(boot_candidates) # Randomly select a boot from the candidate list. boot = random.choice(boot_candidates) return boot
def print_defect_report(): r""" Print a defect report. """ # Making deliberate choice to NOT run plug_in_setup(). We don't want # ffdc_prefix updated. rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( call_point='ffdc_report', stop_on_plug_in_failure=0) # At some point I'd like to have the 'Call FFDC Methods' return a list # of files it has collected. In that case, the following "ls" command # would no longer be needed. For now, however, glob shows the files # named in FFDC_LIST_FILE_PATH so I will refrain from printing those # out (so we don't see duplicates in the list). # Get additional header data which may have been created by ffdc plug-ins. # Also, delete the individual header files to cleanup. cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, show_err=0) # Get additional header data which may have been created by ffdc plug-ins. # Also, delete the individual header files to cleanup. cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, show_err=0) LOG_PREFIX = BuiltIn().get_variable_value("${LOG_PREFIX}") output = '\n'.join(sorted(glob.glob(LOG_PREFIX + '*'))) try: ffdc_list = open(ffdc_list_file_path, 'r') except IOError: ffdc_list = "" # Open ffdc_file_list for writing. We will write a complete list of # FFDC files to it for possible use by plug-ins like cp_stop_check. ffdc_list_file = open(ffdc_list_file_path, 'w') gp.qprintn() # indent=0, width=90, linefeed=1, char="=" gp.qprint_dashes(0, 90, 1, "=") gp.qprintn("Copy this data to the defect:\n") if len(more_header_info) > 0: gp.printn(more_header_info) gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, openbmc_host_name, openbmc_ip, openbmc_username, openbmc_password, os_host, os_host_name, os_ip, os_username, os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, pdu_password, pdu_slot_no, openbmc_serial_host, openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) gp.qprintn() print_last_boots() gp.qprintn() gp.qprint_var(state) gp.qprintn() gp.qprintn("FFDC data files:") if status_file_path != "": gp.qprintn(status_file_path) ffdc_list_file.write(status_file_path + "\n") gp.qprintn(output) # gp.qprintn(ffdc_list) gp.qprintn() if len(ffdc_summary_info) > 0: gp.printn(ffdc_summary_info) gp.qprint_dashes(0, 90, 1, "=") ffdc_list_file.write(output + "\n") ffdc_list_file.close()
def robot_cmd_fnc(robot_cmd_buf, robot_jail=os.environ.get('ROBOT_JAIL', ''), gzip=1): r""" Run the robot command string. This function will set the various PATH variables correctly so that you are running the proper version of all imported files, etc. Description of argument(s): robot_cmd_buf The complete robot command string. robot_jail Indicates that this is to run in "robot jail" meaning without visibility to any apolloxxx import files, programs, etc. gqip This indicates that the log, report and output files produced by robot should be gzipped to save space. """ if not gv.valid_value(robot_cmd_buf): return False # Get globals set by init_robot_test_base_dir_path(). module = sys.modules["__main__"] try: ROBOT_TEST_BASE_DIR_PATH = getattr(module, "ROBOT_TEST_BASE_DIR_PATH") except NameError: init_robot_test_base_dir_path() ROBOT_TEST_BASE_DIR_PATH = getattr(module, "ROBOT_TEST_BASE_DIR_PATH") ROBOT_TEST_RUNNING_FROM_SB = \ gm.get_mod_global("ROBOT_TEST_RUNNING_FROM_SB") if robot_jail == "": if ROBOT_TEST_RUNNING_FROM_SB: robot_jail = 0 else: robot_jail = 1 robot_jail = int(robot_jail) ROBOT_JAIL = os.environ.get('ROBOT_JAIL', '') gp.dprint_vars(ROBOT_TEST_BASE_DIR_PATH, ROBOT_TEST_RUNNING_FROM_SB, ROBOT_JAIL, robot_jail) # Save PATH and PYTHONPATH to be restored later. os.environ["SAVED_PYTHONPATH"] = os.environ.get("PYTHONPATH", "") os.environ["SAVED_PATH"] = os.environ.get("PATH", "") if robot_jail: PYTHONPATH = ROBOT_TEST_BASE_DIR_PATH + "lib" NEW_PATH_LIST = [ROBOT_TEST_BASE_DIR_PATH + "bin"] # Coding special case to preserve python27_path. python27_path = "/opt/rh/python27/root/usr/bin" PATH_LIST = os.environ.get("PATH", "").split(":") if python27_path in PATH_LIST: NEW_PATH_LIST.append(python27_path) NEW_PATH_LIST.extend([ "/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin" ]) PATH = ":".join(NEW_PATH_LIST) else: PYTHONPATH = os.environ.get('PYTHONPATH', '') + ":" +\ ROBOT_TEST_BASE_DIR_PATH + "lib/" PATH = os.environ.get('PATH', '') + ":" + ROBOT_TEST_BASE_DIR_PATH +\ "bin/" os.environ['PYTHONPATH'] = PYTHONPATH os.environ['PATH'] = PATH gp.dprint_vars(PATH, PYTHONPATH) os.environ['FFDC_DIR_PATH_STYLE'] = os.environ.get('FFDC_DIR_PATH_STYLE', '1') test_mode = getattr(module, "test_mode") gp.qpissuing(robot_cmd_buf, test_mode) if test_mode: os.environ["PATH"] = os.environ.get("SAVED_PATH", "") os.environ["PYTHONPATH"] = os.environ.get("SAVED_PYTHONPATH", "") return True if quiet: DEVNULL = open(os.devnull, 'wb') stdout = DEVNULL else: stdout = None sub_proc = subprocess.Popen(robot_cmd_buf, stdout=stdout, shell=True) sub_proc.communicate() shell_rc = sub_proc.returncode if shell_rc != 0: hex = 1 gp.pvar(shell_rc, hex) os.environ["PATH"] = os.environ.get("SAVED_PATH", "") os.environ["PYTHONPATH"] = os.environ.get("SAVED_PYTHONPATH", "") return False os.environ["PATH"] = os.environ.get("SAVED_PATH", "") os.environ["PYTHONPATH"] = os.environ.get("SAVED_PYTHONPATH", "") if not gzip: return True # gzip the output files. # Retrieve the parms from the robot command buffer. robot_cmd_buf_dict = gc.parse_command_string(robot_cmd_buf) # Get prefix from the log parm. prefix = re.sub('log\\.html$', '', robot_cmd_buf_dict['log']) gp.qprintn() rc, outbuf = gc.cmd_fnc("cd " + robot_cmd_buf_dict['outputdir'] + " ; gzip " + robot_cmd_buf_dict['output'] + " " + robot_cmd_buf_dict['log'] + " " + robot_cmd_buf_dict['report']) outputdir = gm.add_trailing_slash(robot_cmd_buf_dict['outputdir']) Output = outputdir + robot_cmd_buf_dict['output'] + ".gz" Log = outputdir + robot_cmd_buf_dict['log'] + ".gz" Report = outputdir + robot_cmd_buf_dict['report'] + ".gz" gp.qprintn("\ngzipped output:") gp.qpvars(0, 9, Output, Log, Report) return True