def print_error_logs(error_logs, key_list=None): r""" Print the error logs to the console screen. This function provides the following benefits: - It will specify print_var parms for the caller (e.g. hex=1). - It is much easier to call this function than to generate the desired code directly from a robot script. Description of argument(s): error_logs An error log dictionary such as the one returned by the 'Get Error Logs' keyword. key_list The list of keys to be printed. This may be specified as either a python list or a space-delimited string. In the latter case, this function will convert it to a python list. See the sprint_varx function prolog for additionatl details. Example use from a python script: ${error_logs}= Get Error Logs Print Error Logs ${error_logs} Message Timestamp Sample output: error_logs: [/xyz/openbmc_project/logging/entry/3]: [Timestamp]: 1521738335735 [Message]: xyz.openbmc_project.Inventory.Error.Nonfunctional [/xyz/openbmc_project/logging/entry/2]: [Timestamp]: 1521738334637 [Message]: xyz.openbmc_project.Inventory.Error.Nonfunctional [/xyz/openbmc_project/logging/entry/1]: [Timestamp]: 1521738300696 [Message]: xyz.openbmc_project.Inventory.Error.Nonfunctional [/xyz/openbmc_project/logging/entry/4]: [Timestamp]: 1521738337915 [Message]: xyz.openbmc_project.Inventory.Error.Nonfunctional Another example call using a robot list: ${error_logs}= Get Error Logs ${key_list}= Create List Message Timestamp Severity Print Error Logs ${error_logs} ${key_list} """ if key_list is not None: if type(key_list) in (str, unicode): key_list = key_list.split(" ") key_list.insert(0, var.BMC_LOGGING_ENTRY + ".*") gp.print_var(error_logs, hex=1, key_list=key_list)
def print_error_logs(error_logs, key_list=None): r""" Print the error logs to the console screen. This function provides the following benefits: - It will specify print_var parms for the caller (e.g. hex=1). - It is much easier to call this function than to generate the desired code directly from a robot script. Description of argument(s): error_logs An error log dictionary such as the one returned by the 'Get Error Logs' keyword. key_list The list of keys to be printed. This may be specified as either a python list or a space-delimited string. In the latter case, this function will convert it to a python list. See the sprint_varx function prolog for additionatl details. Example use from a python script: ${error_logs}= Get Error Logs Print Error Logs ${error_logs} Message Timestamp Sample output: error_logs: [/xyz/openbmc_project/logging/entry/3]: [Timestamp]: 1521738335735 [Message]: xyz.openbmc_project.Inventory.Error.Nonfunctional [/xyz/openbmc_project/logging/entry/2]: [Timestamp]: 1521738334637 [Message]: xyz.openbmc_project.Inventory.Error.Nonfunctional [/xyz/openbmc_project/logging/entry/1]: [Timestamp]: 1521738300696 [Message]: xyz.openbmc_project.Inventory.Error.Nonfunctional [/xyz/openbmc_project/logging/entry/4]: [Timestamp]: 1521738337915 [Message]: xyz.openbmc_project.Inventory.Error.Nonfunctional Another example call using a robot list: ${error_logs}= Get Error Logs ${key_list}= Create List Message Timestamp Severity Print Error Logs ${error_logs} ${key_list} """ if key_list is not None: if type(key_list) in (str, unicode): key_list = key_list.split(" ") key_list.insert(0, var.BMC_LOGGING_ENTRY + ".*") gp.print_var(error_logs, hex=1, key_list=key_list)
def verify_image_upload(image_version, timeout=3): r""" Verify the image was uploaded correctly and that it created a valid d-bus object. If the first check for the image fails, try again until we reach the timeout. Description of argument(s): image_version The version from the image's manifest file (e.g. "IBM-witherspoon-redbud-ibm-OP9_v1.17_1.68"). timeout How long, in minutes, to keep trying to find the image on the BMC. Default is 3 minutes. """ image_path = get_image_path(image_version) image_version_id = image_path.split("/")[-2] keyword.run_key_u("Open Connection And Log In") image_purpose = get_image_purpose(image_path + "MANIFEST") if (image_purpose == var.VERSION_PURPOSE_BMC or image_purpose == var.VERSION_PURPOSE_HOST): uri = var.SOFTWARE_VERSION_URI + image_version_id ret_values = "" for itr in range(timeout * 2): status, ret_values = \ keyword.run_key("Read Attribute " + uri + " Activation") if ((ret_values == var.READY) or (ret_values == var.INVALID) or (ret_values == var.ACTIVE)): return True, image_version_id else: time.sleep(30) # If we exit the for loop, the timeout has been reached gp.print_var(ret_values) return False, None else: gp.print_var(image_purpose) return False, None
def verify_image_upload(image_version, timeout=3): r""" Verify the image was uploaded correctly and that it created a valid d-bus object. If the first check for the image fails, try again until we reach the timeout. Description of argument(s): image_version The version from the image's manifest file (e.g. "v2.2-253-g00050f1"). timeout How long, in minutes, to keep trying to find the image on the BMC. Default is 3 minutes. """ image_path = get_image_path(image_version) image_version_id = image_path.split("/")[-2] keyword.run_key_u("Open Connection And Log In") image_purpose = get_image_purpose(image_path + "MANIFEST") if (image_purpose == var.VERSION_PURPOSE_BMC or image_purpose == var.VERSION_PURPOSE_HOST): uri = var.SOFTWARE_VERSION_URI + image_version_id ret_values = "" for itr in range(timeout * 2): status, ret_values = \ keyword.run_key("Read Attribute " + uri + " Activation") if ((ret_values == var.READY) or (ret_values == var.INVALID) or (ret_values == var.ACTIVE)): return True, image_version_id else: time.sleep(30) # If we exit the for loop, the timeout has been reached gp.print_var(ret_values) return False, None else: gp.print_var(image_purpose) return False, None
def wait_state(match_state=(), wait_time="1 min", interval="1 second", invert=0, openbmc_host="", openbmc_username="", openbmc_password="", os_host="", os_username="", os_password="", quiet=None): r""" Wait for the Open BMC machine's composite state to match the specified state. On success, this keyword returns the machine's composite state as a dictionary. Description of argument(s): match_state A dictionary whose key/value pairs are "state field"/ "state value". See check_state (above) for details. This value may also be any string accepted by return_state_constant (e.g. "standby_match_state"). In such a case this function will call return_state_constant to convert it to a proper dictionary as described above. wait_time The total amount of time to wait for the desired state. This value may be expressed in Robot Framework's time format (e.g. 1 minute, 2 min 3 s, 4.5). interval The amount of time between state checks. This value may be expressed in Robot Framework's time format (e.g. 1 minute, 2 min 3 s, 4.5). invert If this flag is set, this function will for the state of the machine to cease to match the match state. openbmc_host The DNS name or IP address of the BMC. This defaults to global ${OPENBMC_HOST}. openbmc_username The username to be used to login to the BMC. This defaults to global ${OPENBMC_USERNAME}. openbmc_password The password to be used to login to the BMC. This defaults to global ${OPENBMC_PASSWORD}. os_host The DNS name or IP address of the operating system. This defaults to global ${OS_HOST}. os_username The username to be used to login to the OS. This defaults to global ${OS_USERNAME}. os_password The password to be used to login to the OS. This defaults to global ${OS_PASSWORD}. quiet Indicates whether status details should be written to the console. Defaults to either global value of ${QUIET} or to 1. """ quiet = int(gp.get_var_value(quiet, 0)) try: match_state = return_state_constant(match_state) except TypeError: pass if not quiet: if invert: alt_text = "cease to " else: alt_text = "" gp.print_timen("Checking every " + str(interval) + " for up to " + str(wait_time) + " for the state of the machine to " + alt_text + "match the state shown below.") gp.print_var(match_state) if quiet: print_string = "" else: print_string = "#" debug = int(BuiltIn().get_variable_value("${debug}", "0")) if debug: # In debug we print state so no need to print the "#". print_string = "" check_state_quiet = 1 - debug cmd_buf = ["Check State", match_state, "invert=${" + str(invert) + "}", "print_string=" + print_string, "openbmc_host=" + openbmc_host, "openbmc_username="******"openbmc_password="******"os_host=" + os_host, "os_username="******"os_password="******"quiet=${" + str(check_state_quiet) + "}"] gp.dprint_issuing(cmd_buf) try: state = BuiltIn().wait_until_keyword_succeeds(wait_time, interval, *cmd_buf) except AssertionError as my_assertion_error: gp.printn() message = my_assertion_error.args[0] BuiltIn().fail(message) if exit_wait_early_message: # The global exit_wait_early_message was set by a signal handler # indicating that we should fail. message = exit_wait_early_message # Clear the exit_wait_early_message variable for future use. set_exit_wait_early_message("") BuiltIn().fail(gp.sprint_error(message)) if not quiet: gp.printn() if invert: gp.print_timen("The states no longer match:") else: gp.print_timen("The states match:") gp.print_var(state) return state
def check_state(match_state, invert=0, print_string="", openbmc_host="", openbmc_username="", openbmc_password="", os_host="", os_username="", os_password="", quiet=None): r""" Check that the Open BMC machine's composite state matches the specified state. On success, this keyword returns the machine's composite state as a dictionary. Description of argument(s): match_state A dictionary whose key/value pairs are "state field"/ "state value". The state value is interpreted as a regular expression. Example call from robot: ${match_state}= Create Dictionary chassis=^On$ ... bmc=^Ready$ ... boot_progress=^OSStart$ ${state}= Check State &{match_state} invert If this flag is set, this function will succeed if the states do NOT match. print_string This function will print this string to the console prior to getting the state. openbmc_host The DNS name or IP address of the BMC. This defaults to global ${OPENBMC_HOST}. openbmc_username The username to be used to login to the BMC. This defaults to global ${OPENBMC_USERNAME}. openbmc_password The password to be used to login to the BMC. This defaults to global ${OPENBMC_PASSWORD}. os_host The DNS name or IP address of the operating system. This defaults to global ${OS_HOST}. os_username The username to be used to login to the OS. This defaults to global ${OS_USERNAME}. os_password The password to be used to login to the OS. This defaults to global ${OS_PASSWORD}. quiet Indicates whether status details should be written to the console. Defaults to either global value of ${QUIET} or to 1. """ quiet = int(gp.get_var_value(quiet, 0)) gp.gp_print(print_string) try: match_state = return_state_constant(match_state) except TypeError: pass req_states = list(match_state.keys()) # Remove special-case match key from req_states. if expressions_key() in req_states: req_states.remove(expressions_key()) # Initialize state. state = get_state(openbmc_host=openbmc_host, openbmc_username=openbmc_username, openbmc_password=openbmc_password, os_host=os_host, os_username=os_username, os_password=os_password, req_states=req_states, quiet=quiet) if not quiet: gp.print_var(state) if exit_wait_early_message != "": # The exit_wait_early_message has been set by a signal handler so we # will exit "successfully". It is incumbent upon the calling function # (e.g. wait_state) to check/clear this variable and to fail # appropriately. return state match = compare_states(state, match_state) if invert and match: fail_msg = "The current state of the machine matches the match" +\ " state:\n" + gp.sprint_varx("state", state) BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) elif not invert and not match: fail_msg = "The current state of the machine does NOT match the" +\ " match state:\n" +\ gp.sprint_varx("state", state) BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) return state
def check_state(match_state, invert=0, print_string="", openbmc_host="", openbmc_username="", openbmc_password="", os_host="", os_username="", os_password="", quiet=None): r""" Check that the Open BMC machine's composite state matches the specified state. On success, this keyword returns the machine's composite state as a dictionary. Description of arguments: match_state A dictionary whose key/value pairs are "state field"/ "state value". The state value is interpreted as a regular expression. Example call from robot: ${match_state}= Create Dictionary chassis=^On$ ... bmc=^Ready$ ... boot_progress=^OSStart$ ${state}= Check State &{match_state} invert If this flag is set, this function will succeed if the states do NOT match. print_string This function will print this string to the console prior to getting the state. openbmc_host The DNS name or IP address of the BMC. This defaults to global ${OPENBMC_HOST}. openbmc_username The username to be used to login to the BMC. This defaults to global ${OPENBMC_USERNAME}. openbmc_password The password to be used to login to the BMC. This defaults to global ${OPENBMC_PASSWORD}. os_host The DNS name or IP address of the operating system. This defaults to global ${OS_HOST}. os_username The username to be used to login to the OS. This defaults to global ${OS_USERNAME}. os_password The password to be used to login to the OS. This defaults to global ${OS_PASSWORD}. quiet Indicates whether status details should be written to the console. Defaults to either global value of ${QUIET} or to 1. """ quiet = int(gp.get_var_value(quiet, 0)) grp.rprint(print_string) req_states = match_state.keys() # Initialize state. state = get_state(openbmc_host=openbmc_host, openbmc_username=openbmc_username, openbmc_password=openbmc_password, os_host=os_host, os_username=os_username, os_password=os_password, req_states=req_states, quiet=quiet) if not quiet: gp.print_var(state) match = compare_states(state, match_state) if invert and match: fail_msg = "The current state of the machine matches the match" +\ " state:\n" + gp.sprint_varx("state", state) BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) elif not invert and not match: fail_msg = "The current state of the machine does NOT match the" +\ " match state:\n" +\ gp.sprint_varx("state", state) BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) return state
def select_boot(): r""" Select a boot test to be run based on our current state and return the chosen boot type. Description of arguments: state The state of the machine. """ global boot_stack gp.qprint_timen("Selecting a boot test.") my_get_state() stack_popped = 0 if len(boot_stack) > 0: stack_popped = 1 gp.qprint_dashes() gp.qprint_var(boot_stack) gp.qprint_dashes() skip_boot_printed = 0 while len(boot_stack) > 0: boot_candidate = boot_stack.pop() if stack_mode == 'normal': break else: if st.compare_states(state, boot_table[boot_candidate]['end']): if not skip_boot_printed: gp.print_var(stack_mode) gp.printn() gp.print_timen("Skipping the following boot tests" + " which are unnecessary since their" + " required end states match the" + " current machine state:") skip_boot_printed = 1 gp.print_var(boot_candidate) boot_candidate = "" if boot_candidate == "": gp.qprint_dashes() gp.qprint_var(boot_stack) gp.qprint_dashes() return boot_candidate if st.compare_states(state, boot_table[boot_candidate]['start']): gp.qprint_timen("The machine state is valid for a '" + boot_candidate + "' boot test.") gp.qprint_dashes() gp.qprint_var(boot_stack) gp.qprint_dashes() return boot_candidate else: gp.qprint_timen("The machine state does not match the required" + " starting state for a '" + boot_candidate + "' boot test:") gp.print_varx("boot_table[" + boot_candidate + "][start]", boot_table[boot_candidate]['start'], 1) boot_stack.append(boot_candidate) popped_boot = boot_candidate # Loop through your list selecting a boot_candidates boot_candidates = [] for boot_candidate in boot_list: if st.compare_states(state, boot_table[boot_candidate]['start']): if stack_popped: if st.compare_states(boot_table[boot_candidate]['end'], boot_table[popped_boot]['start']): boot_candidates.append(boot_candidate) else: boot_candidates.append(boot_candidate) if len(boot_candidates) == 0: gp.qprint_timen("The user's boot list contained no boot tests" + " which are valid for the current machine state.") boot_candidate = default_power_on if not st.compare_states(state, boot_table[default_power_on]['start']): boot_candidate = default_power_off boot_candidates.append(boot_candidate) gp.qprint_timen("Using default '" + boot_candidate + "' boot type to transition to valid state.") gp.dprint_var(boot_candidates) # Randomly select a boot from the candidate list. boot = random.choice(boot_candidates) return boot
def execute_ssh_command(cmd_buf, open_connection_args={}, login_args={}, print_out=0, print_err=0, ignore_err=1, fork=0, quiet=None, test_mode=None): r""" Run the given command in an SSH session and return the stdout, stderr and the return code. If there is no open SSH connection, this function will connect and login. Likewise, if the caller has not yet logged in to the connection, this function will do the login. NOTE: There is special handling when open_connection_args['alias'] equals "device_connection". - A write, rather than an execute_command, is done. - Only stdout is returned (no stderr or rc). - print_err, ignore_err and fork are not supported. Description of arguments: cmd_buf The command string to be run in an SSH session. open_connection_args A dictionary of arg names and values which are legal to pass to the SSHLibrary open_connection function as parms/args. At a minimum, this should contain a 'host' entry. login_args A dictionary containing the key/value pairs which are acceptable to the SSHLibrary login function as parms/args. At a minimum, this should contain a 'username' and a 'password' entry. print_out If this is set, this function will print the stdout/stderr generated by the shell command. print_err If show_err is set, this function will print a standardized error report if the shell command returns non-zero. ignore_err Indicates that errors encountered on the sshlib.execute_command are to be ignored. fork Indicates that sshlib.start is to be used rather than sshlib.execute_command. quiet Indicates whether this function should run the pissuing() function which prints an "Issuing: <cmd string>" to stdout. This defaults to the global quiet value. test_mode If test_mode is set, this function will not actually run the command. This defaults to the global test_mode value. """ gp.lprint_executing() # Obtain default values. quiet = int(gp.get_var_value(quiet, 0)) test_mode = int(gp.get_var_value(test_mode, 0)) if not quiet: gp.pissuing(cmd_buf, test_mode) gp.lpissuing(cmd_buf, test_mode) if test_mode: return "", "", 0 global sshlib max_exec_cmd_attempts = 2 # Look for existing SSH connection. # Prepare a search connection dictionary. search_connection_args = open_connection_args.copy() # Remove keys that don't work well for searches. search_connection_args.pop("timeout", None) connection = find_connection(search_connection_args) if connection: gp.lprint_timen("Found the following existing connection:") gp.lprintn(sprint_connection(connection)) if connection.alias == "": index_or_alias = connection.index else: index_or_alias = connection.alias gp.lprint_timen("Switching to existing connection: \"" + str(index_or_alias) + "\".") sshlib.switch_connection(index_or_alias) else: gp.lprint_timen("Connecting to " + open_connection_args['host'] + ".") cix = sshlib.open_connection(**open_connection_args) try: login_ssh(login_args) except Exception as login_exception: except_type, except_value, except_traceback = sys.exc_info() rc = 1 stderr = str(except_value) stdout = "" max_exec_cmd_attempts = 0 for exec_cmd_attempt_num in range(1, max_exec_cmd_attempts + 1): gp.lprint_var(exec_cmd_attempt_num) try: if fork: sshlib.start_command(cmd_buf) else: if open_connection_args['alias'] == "device_connection": stdout = sshlib.write(cmd_buf) stderr = "" rc = 0 else: stdout, stderr, rc = \ sshlib.execute_command(cmd_buf, return_stdout=True, return_stderr=True, return_rc=True) except Exception as execute_exception: except_type, except_value, except_traceback = sys.exc_info() gp.lprint_var(except_type) gp.lprint_varx("except_value", str(except_value)) if except_type is exceptions.AssertionError and\ re.match(r"Connection not open", str(except_value)): login_ssh(login_args) # Now we must continue to next loop iteration to retry the # execute_command. continue if (except_type is paramiko.ssh_exception.SSHException and re.match(r"SSH session not active", str(except_value))) or\ (except_type is socket.error and re.match(r"\[Errno 104\] Connection reset by peer", str(except_value))): # Close and re-open a connection. # Note: close_connection() doesn't appear to get rid of the # connection. It merely closes it. Since there is a concern # about over-consumption of resources, we use # close_all_connections() which also gets rid of all # connections. gp.lprint_timen("Closing all connections.") sshlib.close_all_connections() gp.lprint_timen("Connecting to " + open_connection_args['host'] + ".") cix = sshlib.open_connection(**open_connection_args) login_ssh(login_args) continue # We do not handle any other RuntimeErrors so we will raise the # exception again. sshlib.close_all_connections() raise(execute_exception) # If we get to this point, the command was executed. break if fork: return if rc != 0 and print_err: gp.print_var(rc, 1) if not print_out: gp.print_var(stderr) gp.print_var(stdout) if print_out: gp.printn(stderr + stdout) if not ignore_err: message = gp.sprint_error("The prior SSH" + " command returned a non-zero return" + " code:\n" + gp.sprint_var(rc, 1) + stderr + "\n") BuiltIn().should_be_equal(rc, 0, message) if open_connection_args['alias'] == "device_connection": return stdout return stdout, stderr, rc
def select_boot(): r""" Select a boot test to be run based on our current state and return the chosen boot type. Description of arguments: state The state of the machine. """ global boot_stack gp.qprint_timen("Selecting a boot test.") my_get_state() stack_popped = 0 if len(boot_stack) > 0: stack_popped = 1 gp.qprint_dashes() gp.qprint_var(boot_stack) gp.qprint_dashes() skip_boot_printed = 0 while len(boot_stack) > 0: boot_candidate = boot_stack.pop() if stack_mode == 'normal': break else: if st.compare_states(state, boot_table[boot_candidate]['end']): if not skip_boot_printed: gp.print_var(stack_mode) gp.printn() gp.print_timen("Skipping the following boot tests" + " which are unnecessary since their" + " required end states match the" + " current machine state:") skip_boot_printed = 1 gp.print_var(boot_candidate) boot_candidate = "" if boot_candidate == "": gp.qprint_dashes() gp.qprint_var(boot_stack) gp.qprint_dashes() return boot_candidate if st.compare_states(state, boot_table[boot_candidate]['start']): gp.qprint_timen("The machine state is valid for a '" + boot_candidate + "' boot test.") gp.qprint_dashes() gp.qprint_var(boot_stack) gp.qprint_dashes() return boot_candidate else: gp.qprint_timen("The machine state does not match the required" + " starting state for a '" + boot_candidate + "' boot test:") gp.print_varx("boot_table[" + boot_candidate + "][start]", boot_table[boot_candidate]['start'], 1) boot_stack.append(boot_candidate) popped_boot = boot_candidate # Loop through your list selecting a boot_candidates boot_candidates = [] for boot_candidate in boot_list: if st.compare_states(state, boot_table[boot_candidate]['start']): if stack_popped: if st.compare_states(boot_table[boot_candidate]['end'], boot_table[popped_boot]['start']): boot_candidates.append(boot_candidate) else: boot_candidates.append(boot_candidate) if len(boot_candidates) == 0: gp.qprint_timen("The user's boot list contained no boot tests" + " which are valid for the current machine state.") boot_candidate = default_power_on if not st.compare_states(state, boot_table[default_power_on]['start']): boot_candidate = default_power_off boot_candidates.append(boot_candidate) gp.qprint_timen("Using default '" + boot_candidate + "' boot type to transition to valid state.") gp.dprint_var(boot_candidates) # Randomly select a boot from the candidate list. boot = random.choice(boot_candidates) return boot
def rprocess_plug_in_packages(plug_in_packages_list=None, call_point="setup", shell_rc="0x00000000", stop_on_plug_in_failure=1, stop_on_non_zero_rc=0, release_type="obmc", quiet=None, debug=None, return_history=False): r""" Call the external process_plug_in_packages.py to process the plug-in packages. Return the following: rc The return code - 0 = PASS, 1 = FAIL. shell_rc The shell return code returned by process_plug_in_packages.py. failed_plug_in_name The failed plug in name (if any). Description of arguments: plug_in_packages_list A python list of plug-in directory paths. call_point The call point program to be called for each plug-in package (e.g. post_boot). This name should not include the "cp_" prefix. shell_rc The user may supply a value other than zero to indicate an acceptable non-zero return code. For example, if this value equals 0x00000200, it means that for each plug-in call point that runs, a 0x00000200 will not be counted as a failure. stop_on_plug_in_failure If this parameter is set to 1, this program will stop and return non-zero if the call point program from any plug-in directory fails. Conversely, if it is set to false, this program will run the call point program from each and every plug-in directory regardless of their return values. Typical example cases where you'd want to run all plug-in call points regardless of success or failure would be "cleanup" or "ffdc" call points. stop_on_non_zero_rc If this parm is set to 1 and a plug-in call point program returns a valid non-zero return code (see "shell_rc" parm above), this program will stop processing and return 0 (success). Since this constitutes a successful exit, this would normally be used where the caller wishes to stop processing if one of the plug-in directory call point programs returns a special value indicating that some special case has been found. An example might be in calling some kind of "check_errl" call point program. Such a call point program might return a 2 (i.e. 0x00000200) to indicate that a given error log entry was found in an "ignore" list and is therefore to be ignored. That being the case, no other "check_errl" call point program would need to be called. release_type The type of release being tested (e.g. "obmc", "op", "fips"). This influences which integrated plug-ins are selected. quiet If quiet is set to 1, this function will NOT write status messages to stdout. This will default to the global quiet program parm or to 0. debug If this parameter is set to 1, this function will print additional debug information. This is mainly to be used by the developer of this function. This will default to the global quiet program parm or to 0. return_history In addition to rc, shell_rc and failed_plug_in_name, return a list containing historical output that looks like the following: history: history[0]: #(CDT) 2018/10/30 12:25:49 - Running OBMC_Sample/cp_post_stack """ rc = 0 plug_in_packages_list = gp.get_var_value(plug_in_packages_list, []) # If there are no plug-in packages to process, return successfully. if len(plug_in_packages_list) == 0: if return_history: return 0, 0, "", [] else: return 0, 0, "" quiet = int(gp.get_var_value(quiet, 0)) debug = int(gp.get_var_value(debug, 0)) # Create string from list. plug_in_dir_paths = ':'.join(plug_in_packages_list) temp = tempfile.NamedTemporaryFile() temp_file_path = temp.name temp2 = tempfile.NamedTemporaryFile() temp_properties_file_path = temp2.name if debug: os.environ["PERF_TRACE"] = "1" debug_string = " --quiet=0" else: debug_string = "" loc_shell_rc = 0 sub_cmd_buf = "process_plug_in_packages.py" + debug_string +\ " --call_point=" + call_point + " --allow_shell_rc=" +\ str(shell_rc) + " --stop_on_plug_in_failure=" +\ str(stop_on_plug_in_failure) + " --stop_on_non_zero_rc=" +\ str(stop_on_non_zero_rc) + " " + plug_in_dir_paths if quiet: cmd_buf = sub_cmd_buf + " > " + temp_file_path + " 2>&1" else: cmd_buf = "set -o pipefail ; " + sub_cmd_buf + " 2>&1 | tee " +\ temp_file_path if debug: gp.print_issuing(cmd_buf) else: gp.print_timen("Processing " + call_point + " call point programs.") sub_proc = subprocess.Popen(cmd_buf, shell=True, executable='/bin/bash') sub_proc.communicate() proc_plug_pkg_rc = sub_proc.returncode if return_history: # Get the "Running" statements from the output. regex = " Running [^/]+/cp_" cmd_buf = "egrep '" + regex + "' " + temp_file_path _, history = gc.shell_cmd(cmd_buf, quiet=(not debug), print_output=0, show_err=0, ignore_err=1) history = [x + "\n" for x in filter(None, history.split("\n"))] else: history = [] # As process_plug_in_packages.py help text states, it will print the values of failed_plug_in_name and # shell_rc in the following format: # failed_plug_in_name: <failed plug-in value, if any> # shell_rc: <shell return code value of last call point program> # We want to obtain those values from the output. To make the task simpler, we'll start by grepping the # output for lines that might fit such a format: # A valid bash variable against the left margin followed by... # - A colon followed by... # - Zero or more spaces bash_var_regex = "[_[:alpha:]][_[:alnum:]]*" regex = "^" + bash_var_regex + ":[ ]*" cmd_buf = "egrep '" + regex + "' " + temp_file_path + " > " +\ temp_properties_file_path gp.dprint_issuing(cmd_buf) grep_rc = os.system(cmd_buf) # Next we call my_parm_file to create a properties dictionary. properties = gm.my_parm_file(temp_properties_file_path) # Finally, we access the 2 values that we need. shell_rc = int(properties.get('shell_rc', '0x0000000000000000'), 16) failed_plug_in_name = properties.get('failed_plug_in_name', '') if proc_plug_pkg_rc != 0: if quiet: os.system("cat " + temp_file_path + " >&2") if grep_rc != 0: gp.print_var(grep_rc, gp.hexa()) gp.print_var(proc_plug_pkg_rc, gp.hexa()) gp.print_timen("Re-cap of plug-in failures:") gc.cmd_fnc_u("egrep -A 1 '^failed_plug_in_name:[ ]+' " + temp_properties_file_path + " | egrep -v '^\\--'", quiet=1, show_err=0) rc = 1 if return_history: return rc, shell_rc, failed_plug_in_name, history else: return rc, shell_rc, failed_plug_in_name
def robot_cmd_fnc(robot_cmd_buf, robot_jail=os.environ.get('ROBOT_JAIL', ''), gzip=1): r""" Run the robot command string. This function will set the various PATH variables correctly so that you are running the proper version of all imported files, etc. Description of argument(s): robot_cmd_buf The complete robot command string. robot_jail Indicates that this is to run in "robot jail" meaning without visibility to any apolloxxx import files, programs, etc. gqip This indicates that the log, report and output files produced by robot should be gzipped to save space. """ if not gv.valid_value(robot_cmd_buf): return False # Set global variables to aid in cleanup with process_robot_output_files. global gcr_last_robot_cmd_buf global gcr_last_robot_rc gcr_last_robot_cmd_buf = robot_cmd_buf # Get globals set by init_robot_test_base_dir_path(). module = sys.modules["__main__"] try: ROBOT_TEST_BASE_DIR_PATH = getattr(module, "ROBOT_TEST_BASE_DIR_PATH") except NameError: init_robot_test_base_dir_path() ROBOT_TEST_BASE_DIR_PATH = getattr(module, "ROBOT_TEST_BASE_DIR_PATH") ROBOT_TEST_RUNNING_FROM_SB = \ gm.get_mod_global("ROBOT_TEST_RUNNING_FROM_SB") if robot_jail == "": if ROBOT_TEST_RUNNING_FROM_SB: robot_jail = 0 else: robot_jail = 1 robot_jail = int(robot_jail) ROBOT_JAIL = os.environ.get('ROBOT_JAIL', '') gp.dprint_vars(ROBOT_TEST_BASE_DIR_PATH, ROBOT_TEST_RUNNING_FROM_SB, ROBOT_JAIL, robot_jail) # Save PATH and PYTHONPATH to be restored later. os.environ["SAVED_PYTHONPATH"] = os.environ.get("PYTHONPATH", "") os.environ["SAVED_PATH"] = os.environ.get("PATH", "") if robot_jail: PYTHONPATH = ROBOT_TEST_BASE_DIR_PATH + "lib" NEW_PATH_LIST = [ROBOT_TEST_BASE_DIR_PATH + "bin"] # Coding special case to preserve python27_path. python27_path = "/opt/rh/python27/root/usr/bin" PATH_LIST = os.environ.get("PATH", "").split(":") if python27_path in PATH_LIST: NEW_PATH_LIST.append(python27_path) NEW_PATH_LIST.extend(["/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin"]) PATH = ":".join(NEW_PATH_LIST) else: PYTHONPATH = os.environ.get('PYTHONPATH', '') + ":" +\ ROBOT_TEST_BASE_DIR_PATH + "lib/" PATH = os.environ.get('PATH', '') + ":" + ROBOT_TEST_BASE_DIR_PATH +\ "bin/" os.environ['PYTHONPATH'] = PYTHONPATH os.environ['PATH'] = PATH gp.dprint_vars(PATH, PYTHONPATH) os.environ['FFDC_DIR_PATH_STYLE'] = os.environ.get('FFDC_DIR_PATH_STYLE', '1') test_mode = getattr(module, "test_mode") gp.qpissuing(robot_cmd_buf, test_mode) if test_mode: os.environ["PATH"] = os.environ.get("SAVED_PATH", "") os.environ["PYTHONPATH"] = os.environ.get("SAVED_PYTHONPATH", "") return True if quiet: DEVNULL = open(os.devnull, 'wb') stdout = DEVNULL else: stdout = None sub_proc = subprocess.Popen(robot_cmd_buf, stdout=stdout, shell=True) sub_proc.communicate() shell_rc = sub_proc.returncode os.environ["PATH"] = os.environ.get("SAVED_PATH", "") os.environ["PYTHONPATH"] = os.environ.get("SAVED_PYTHONPATH", "") gcr_last_robot_rc = shell_rc process_robot_output_files() if shell_rc != 0: hex = 1 gp.print_var(shell_rc, hex) return False return True
def robot_cmd_fnc(robot_cmd_buf, robot_jail=os.environ.get('ROBOT_JAIL', ''), quiet=None, test_mode=0): r""" Run the robot command string. This function will set the various PATH variables correctly so that you are running the proper version of all imported files, etc. Description of argument(s): robot_cmd_buf The complete robot command string. robot_jail Indicates that this is to run in "robot jail" meaning without visibility to any apolloxxx import files, programs, etc. test_mode If test_mode is set, this function will not actually run the command. """ quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0))) gv.valid_value(robot_cmd_buf) # Set global variables to aid in cleanup with process_robot_output_files. global gcr_last_robot_cmd_buf global gcr_last_robot_rc gcr_last_robot_cmd_buf = robot_cmd_buf # Get globals set by init_robot_test_base_dir_path(). module = sys.modules["__main__"] try: ROBOT_TEST_BASE_DIR_PATH = getattr(module, "ROBOT_TEST_BASE_DIR_PATH") except NameError: init_robot_test_base_dir_path() ROBOT_TEST_BASE_DIR_PATH = getattr(module, "ROBOT_TEST_BASE_DIR_PATH") ROBOT_TEST_RUNNING_FROM_SB = gm.get_mod_global( "ROBOT_TEST_RUNNING_FROM_SB") OPENBMCTOOL_DIR_PATH = gm.get_mod_global("OPENBMCTOOL_DIR_PATH") if robot_jail == "": if ROBOT_TEST_RUNNING_FROM_SB: robot_jail = 0 else: robot_jail = 1 robot_jail = int(robot_jail) ROBOT_JAIL = os.environ.get('ROBOT_JAIL', '') gp.dprint_vars(ROBOT_TEST_BASE_DIR_PATH, ROBOT_TEST_RUNNING_FROM_SB, ROBOT_JAIL, robot_jail) # Save PATH and PYTHONPATH to be restored later. os.environ["SAVED_PYTHONPATH"] = os.environ.get("PYTHONPATH", "") os.environ["SAVED_PATH"] = os.environ.get("PATH", "") if robot_jail: # Make sure required programs like python and robot can be found in the new restricted PATH. required_programs = "python robot" # It is expected that there will be a "python" program in the tool base bin path which is really a # link to select_version. Ditto for "robot". Call each with the --print_only option to get the # paths to the "real" programs. cmd_buf = "for program in " + required_programs \ + " ; do dirname $(${program} --print_only) ; done 2>/dev/null" rc, out_buf = gc.shell_cmd(cmd_buf, quiet=1, print_output=0) PYTHONPATH = ROBOT_TEST_BASE_DIR_PATH + "lib" NEW_PATH_LIST = [ROBOT_TEST_BASE_DIR_PATH + "bin"] NEW_PATH_LIST.extend(list(set(out_buf.rstrip("\n").split("\n")))) NEW_PATH_LIST.extend([ "/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin", OPENBMCTOOL_DIR_PATH.rstrip('/') ]) PATH = ":".join(NEW_PATH_LIST) else: PYTHONPATH = os.environ.get('PYTHONPATH', '') + ":" +\ ROBOT_TEST_BASE_DIR_PATH + "lib" PATH = os.environ.get('PATH', '') + ":" + ROBOT_TEST_BASE_DIR_PATH +\ "bin" + ":" + OPENBMCTOOL_DIR_PATH.rstrip('/') os.environ['PYTHONPATH'] = PYTHONPATH os.environ['PATH'] = PATH gp.dprint_vars(PATH, PYTHONPATH) os.environ['FFDC_DIR_PATH_STYLE'] = os.environ.get('FFDC_DIR_PATH_STYLE', '1') gp.qpissuing(robot_cmd_buf, test_mode) if test_mode: os.environ["PATH"] = os.environ.get("SAVED_PATH", "") os.environ["PYTHONPATH"] = os.environ.get("SAVED_PYTHONPATH", "") return True if quiet: DEVNULL = open(os.devnull, 'wb') stdout = DEVNULL else: stdout = None sub_proc = subprocess.Popen(robot_cmd_buf, stdout=stdout, shell=True) sub_proc.communicate() shell_rc = sub_proc.returncode os.environ["PATH"] = os.environ.get("SAVED_PATH", "") os.environ["PYTHONPATH"] = os.environ.get("SAVED_PYTHONPATH", "") gcr_last_robot_rc = shell_rc process_robot_output_files() if shell_rc != 0: gp.print_var(shell_rc, gp.hexa()) return False return True
def execute_ssh_command(cmd_buf, open_connection_args={}, login_args={}, print_out=0, print_err=0, ignore_err=1, fork=0, quiet=None, test_mode=None, time_out=None): r""" Run the given command in an SSH session and return the stdout, stderr and the return code. If there is no open SSH connection, this function will connect and login. Likewise, if the caller has not yet logged in to the connection, this function will do the login. NOTE: There is special handling when open_connection_args['alias'] equals "device_connection". - A write, rather than an execute_command, is done. - Only stdout is returned (no stderr or rc). - print_err, ignore_err and fork are not supported. Description of arguments: cmd_buf The command string to be run in an SSH session. open_connection_args A dictionary of arg names and values which are legal to pass to the SSHLibrary open_connection function as parms/args. At a minimum, this should contain a 'host' entry. login_args A dictionary containing the key/value pairs which are acceptable to the SSHLibrary login function as parms/args. At a minimum, this should contain a 'username' and a 'password' entry. print_out If this is set, this function will print the stdout/stderr generated by the shell command. print_err If show_err is set, this function will print a standardized error report if the shell command returns non-zero. ignore_err Indicates that errors encountered on the sshlib.execute_command are to be ignored. fork Indicates that sshlib.start is to be used rather than sshlib.execute_command. quiet Indicates whether this function should run the pissuing() function which prints an "Issuing: <cmd string>" to stdout. This defaults to the global quiet value. test_mode If test_mode is set, this function will not actually run the command. This defaults to the global test_mode value. time_out The amount of time to allow for the execution of cmd_buf. A value of None means that there is no limit to how long the command may take. """ gp.lprint_executing() # Obtain default values. quiet = int(gp.get_var_value(quiet, 0)) test_mode = int(gp.get_var_value(test_mode, 0)) if not quiet: gp.pissuing(cmd_buf, test_mode) gp.lpissuing(cmd_buf, test_mode) if test_mode: return "", "", 0 global sshlib max_exec_cmd_attempts = 2 # Look for existing SSH connection. # Prepare a search connection dictionary. search_connection_args = open_connection_args.copy() # Remove keys that don't work well for searches. search_connection_args.pop("timeout", None) connection = find_connection(search_connection_args) if connection: gp.lprint_timen("Found the following existing connection:") gp.lprintn(sprint_connection(connection)) if connection.alias == "": index_or_alias = connection.index else: index_or_alias = connection.alias gp.lprint_timen("Switching to existing connection: \"" + str(index_or_alias) + "\".") sshlib.switch_connection(index_or_alias) else: gp.lprint_timen("Connecting to " + open_connection_args['host'] + ".") cix = sshlib.open_connection(**open_connection_args) try: login_ssh(login_args) except Exception: except_type, except_value, except_traceback = sys.exc_info() rc = 1 stderr = str(except_value) stdout = "" max_exec_cmd_attempts = 0 for exec_cmd_attempt_num in range(1, max_exec_cmd_attempts + 1): gp.lprint_var(exec_cmd_attempt_num) try: if fork: sshlib.start_command(cmd_buf) else: if open_connection_args['alias'] == "device_connection": stdout = sshlib.write(cmd_buf) stderr = "" rc = 0 else: stdout, stderr, rc = \ func_timer.run(sshlib.execute_command, cmd_buf, return_stdout=True, return_stderr=True, return_rc=True, time_out=time_out) except Exception: except_type, except_value, except_traceback = sys.exc_info() gp.lprint_var(except_type) gp.lprint_varx("except_value", str(except_value)) # This may be our last time through the retry loop, so setting # return variables. rc = 1 stderr = str(except_value) stdout = "" if except_type is exceptions.AssertionError and\ re.match(r"Connection not open", str(except_value)): try: login_ssh(login_args) # Now we must continue to next loop iteration to retry the # execute_command. continue except Exception: except_type, except_value, except_traceback =\ sys.exc_info() rc = 1 stderr = str(except_value) stdout = "" break if (except_type is paramiko.ssh_exception.SSHException and re.match(r"SSH session not active", str(except_value))) or\ (except_type is socket.error and re.match(r"\[Errno 104\] Connection reset by peer", str(except_value))) or\ (except_type is paramiko.ssh_exception.SSHException and re.match(r"Timeout opening channel\.", str(except_value))): # Close and re-open a connection. # Note: close_connection() doesn't appear to get rid of the # connection. It merely closes it. Since there is a concern # about over-consumption of resources, we use # close_all_connections() which also gets rid of all # connections. gp.lprint_timen("Closing all connections.") sshlib.close_all_connections() gp.lprint_timen("Connecting to " + open_connection_args['host'] + ".") cix = sshlib.open_connection(**open_connection_args) login_ssh(login_args) continue # We do not handle any other RuntimeErrors so we will raise the # exception again. sshlib.close_all_connections() gp.lprintn(traceback.format_exc()) raise (except_value) # If we get to this point, the command was executed. break if fork: return if rc != 0 and print_err: gp.print_var(rc, gp.hexa()) if not print_out: gp.print_var(stderr) gp.print_var(stdout) if print_out: gp.printn(stderr + stdout) if not ignore_err: message = gp.sprint_error("The prior SSH" + " command returned a non-zero return" + " code:\n" + gp.sprint_var(rc, gp.hexa()) + stderr + "\n") BuiltIn().should_be_equal(rc, 0, message) if open_connection_args['alias'] == "device_connection": return stdout return stdout, stderr, rc
def execute_ssh_command(cmd_buf, open_connection_args={}, login_args={}, print_out=0, print_err=0, ignore_err=1, fork=0, quiet=None, test_mode=None): r""" Run the given command in an SSH session and return the stdout, stderr and the return code. If there is no open SSH connection, this function will connect and login. Likewise, if the caller has not yet logged in to the connection, this function will do the login. Description of arguments: cmd_buf The command string to be run in an SSH session. open_connection_args A dictionary of arg names and values which are legal to pass to the SSHLibrary open_connection function as parms/args. At a minimum, this should contain a 'host' entry. login_args A dictionary containing the key/value pairs which are acceptable to the SSHLibrary login function as parms/args. At a minimum, this should contain a 'username' and a 'password' entry. print_out If this is set, this function will print the stdout/stderr generated by the shell command. print_err If show_err is set, this function will print a standardized error report if the shell command returns non-zero. ignore_err Indicates that errors encountered on the sshlib.execute_command are to be ignored. fork Indicates that sshlib.start is to be used rather than sshlib.execute_command. quiet Indicates whether this function should run the pissuing() function which prints an "Issuing: <cmd string>" to stdout. This defaults to the global quiet value. test_mode If test_mode is set, this function will not actually run the command. This defaults to the global test_mode value. """ gp.dprint_executing() # Obtain default values. quiet = int(gp.get_var_value(quiet, 0)) test_mode = int(gp.get_var_value(test_mode, 0)) if not quiet: gp.pissuing(cmd_buf, test_mode) if test_mode: return "", "", 0 global sshlib # Look for existing SSH connection. # Prepare a search connection dictionary. search_connection_args = open_connection_args.copy() # Remove keys that don't work well for searches. search_connection_args.pop("timeout", None) connection = find_connection(search_connection_args) if connection: gp.dprint_timen("Found the following existing connection:") gp.dprintn(sprint_connection(connection)) if connection.alias == "": index_or_alias = connection.index else: index_or_alias = connection.alias gp.dprint_timen("Switching to existing connection: \"" + str(index_or_alias) + "\".") sshlib.switch_connection(index_or_alias) else: gp.dprint_timen("Connecting to " + open_connection_args['host'] + ".") cix = sshlib.open_connection(**open_connection_args) login_ssh(login_args) max_exec_cmd_attempts = 2 for exec_cmd_attempt_num in range(1, max_exec_cmd_attempts + 1): gp.dprint_var(exec_cmd_attempt_num) try: if fork: sshlib.start_command(cmd_buf) else: stdout, stderr, rc = sshlib.execute_command(cmd_buf, return_stdout=True, return_stderr=True, return_rc=True) except Exception as execute_exception: except_type, except_value, except_traceback = sys.exc_info() gp.dprint_var(except_type) gp.dprint_varx("except_value", str(except_value)) if except_type is exceptions.AssertionError and\ re.match(r"Connection not open", str(except_value)): login_ssh(login_args) # Now we must continue to next loop iteration to retry the # execute_command. continue if except_type is paramiko.ssh_exception.SSHException and\ re.match(r"SSH session not active", str(except_value)): # Close and re-open a connection. sshlib.close_connection() gp.dprint_timen("Connecting to " + open_connection_args['host'] + ".") cix = sshlib.open_connection(**open_connection_args) login_ssh(login_args) continue # We do not handle any other RuntimeErrors so we will raise the # exception again. raise(execute_exception) # If we get to this point, the command was executed. break if fork: return if rc != 0 and print_err: gp.print_var(rc, 1) if not print_out: gp.print_var(stderr) gp.print_var(stdout) if print_out: gp.printn(stderr + stdout) if not ignore_err: message = gp.sprint_error("The prior SSH" + " command returned a non-zero return" + " code:\n" + gp.sprint_var(rc, 1) + stderr + "\n") BuiltIn().should_be_equal(rc, 0, message) return stdout, stderr, rc