def get_dump_dict(quiet=None): r""" Get dump information and return as an ordered dictionary where the keys are the dump IDs and the values are the full path names of the dumps. Example robot program call: ${dump_dict}= Get Dump Dict Rpvars 1 dump_dict Example output: dump_dict: [1]: /var/lib/phosphor-debug-collector/dumps/1/obmcdump_1_1508255216.tar.xz [2]: /var/lib/phosphor-debug-collector/dumps/2/obmcdump_2_1508255245.tar.xz [3]: /var/lib/phosphor-debug-collector/dumps/3/obmcdump_3_1508255267.tar.xz [4]: /var/lib/phosphor-debug-collector/dumps/4/obmcdump_4_1508255283.tar.xz Description of argument(s): quiet If quiet is set to 1, this function will NOT write status messages to stdout. """ quiet = int(gp.get_var_value(quiet, 1)) cmd_buf = "dump_dir_path=" + var.DUMP_DIR_PATH + " ; " \ + "for dump_id in $(ls ${dump_dir_path} | sort -n) ; " \ + "do echo -n $dump_id: ; ls ${dump_dir_path}${dump_id}/* ; done" output, stderr, rc = bsu.bmc_execute_command(cmd_buf, quiet=quiet) return vf.key_value_outbuf_to_dict(output)
def get_dump_dict(quiet=None): r""" Get dump information and return as an ordered dictionary where the keys are the dump IDs and the values are the full path names of the dumps. Example robot program call: ${dump_dict}= Get Dump Dict Rprint Vars dump_dict Example output: dump_dict: [1]: /var/lib/phosphor-debug-collector/dumps/1/obmcdump_1_1508255216.tar.xz [2]: /var/lib/phosphor-debug-collector/dumps/2/obmcdump_2_1508255245.tar.xz [3]: /var/lib/phosphor-debug-collector/dumps/3/obmcdump_3_1508255267.tar.xz [4]: /var/lib/phosphor-debug-collector/dumps/4/obmcdump_4_1508255283.tar.xz Description of argument(s): quiet If quiet is set to 1, this function will NOT write status messages to stdout. """ quiet = int(gp.get_var_value(quiet, 1)) cmd_buf = "dump_dir_path=" + var.DUMP_DIR_PATH + " ; " \ + "for dump_id in $(ls ${dump_dir_path} | sort -n) ; do " \ + "file_path=$(ls ${dump_dir_path}${dump_id}/* 2>/dev/null)" \ + " || continue ; echo ${dump_id}:${file_path} ; done" output, stderr, rc = bsu.bmc_execute_command(cmd_buf, quiet=quiet) return vf.key_value_outbuf_to_dict(output)
def wait_for_comm_cycle(start_boot_seconds, quiet=None): r""" Wait for communications to the BMC to stop working and then resume working. This function is useful when you have initiated some kind of reboot. Description of arguments: start_boot_seconds The time that the boot test started. The format is the epoch time in seconds, i.e. the number of seconds since 1970-01-01 00:00:00 UTC. This value should be obtained from the BMC so that it is not dependent on any kind of synchronization between this machine and the target BMC This will allow this program to work correctly even in a simulated environment. This value should be obtained by the caller prior to initiating a reboot. It can be obtained as follows: state = st.get_state(req_states=['epoch_seconds']) """ quiet = int(gp.get_var_value(quiet, 0)) # Validate parms. error_message = gv.svalid_integer(start_boot_seconds, var_name="start_boot_seconds") if error_message != "": BuiltIn().fail(gp.sprint_error(error_message)) match_state = anchor_state(DotDict([('packet_loss', '100')])) # Wait for 100% packet loss trying to ping machine. wait_state(match_state, wait_time="8 mins", interval="0 seconds") match_state['packet_loss'] = '^0$' # Wait for 0% packet loss trying to ping machine. wait_state(match_state, wait_time="8 mins", interval="0 seconds") # Get the uptime and epoch seconds for comparisons. We want to be sure # that the uptime is less than the elapsed boot time. Further proof that # a reboot has indeed occurred (vs random network instability giving a # false positive. state = get_state(req_states=['uptime', 'epoch_seconds'], quiet=quiet) elapsed_boot_time = int(state['epoch_seconds']) - start_boot_seconds gp.qprint_var(elapsed_boot_time) if int(float(state['uptime'])) < elapsed_boot_time: uptime = state['uptime'] gp.qprint_var(uptime) gp.qprint_timen("The uptime is less than the elapsed boot time," + " as expected.") else: error_message = "The uptime is greater than the elapsed boot time," +\ " which is unexpected:\n" +\ gp.sprint_var(start_boot_seconds) +\ gp.sprint_var(state) BuiltIn().fail(gp.sprint_error(error_message)) gp.qprint_timen("Verifying that REST API interface is working.") match_state = DotDict([('rest', '^1$')]) state = wait_state(match_state, wait_time="5 mins", interval="2 seconds")
def t_shell_cmd(command_string, **kwargs): r""" Search upward in the the call stack to obtain the test_mode argument, add it to kwargs and then call shell_cmd and return the result. See shell_cmd prolog for details on all arguments. """ if 'test_mode' in kwargs: error_message = "Programmer error - test_mode is not a valid" +\ " argument to this function." gp.print_error_report(error_message) exit(1) test_mode = gp.get_stack_var('test_mode', int(gp.get_var_value(None, 0, "test_mode"))) kwargs['test_mode'] = test_mode return shell_cmd(command_string, **kwargs)
def wait_for_comm_cycle(start_boot_seconds, quiet=None): r""" Wait for the BMC uptime to be less than elapsed_boot_time. This function will tolerate an expected loss of communication to the BMC. This function is useful when some kind of reboot has been initiated by the caller. Description of argument(s): start_boot_seconds The time that the boot test started. The format is the epoch time in seconds, i.e. the number of seconds since 1970-01-01 00:00:00 UTC. This value should be obtained from the BMC so that it is not dependent on any kind of synchronization between this machine and the target BMC This will allow this program to work correctly even in a simulated environment. This value should be obtained by the caller prior to initiating a reboot. It can be obtained as follows: state = st.get_state(req_states=['epoch_seconds']) """ quiet = int(gp.get_var_value(quiet, 0)) # Validate parms. error_message = gv.valid_integer(start_boot_seconds) if error_message: BuiltIn().fail(gp.sprint_error(error_message)) # Wait for uptime to be less than elapsed_boot_time. set_start_boot_seconds(start_boot_seconds) expr = 'int(float(state[\'uptime\'])) < int(state[\'elapsed_boot_time\'])' match_state = DotDict([('uptime', '^[0-9\\.]+$'), ('elapsed_boot_time', '^[0-9]+$'), (expressions_key(), [expr])]) wait_state(match_state, wait_time="12 mins", interval="5 seconds") gp.qprint_timen("Verifying that REST/Redfish API interface is working.") if not redfish_support_trans_state: match_state = DotDict([('rest', '^1$')]) else: match_state = DotDict([('redfish', '^1$')]) state = wait_state(match_state, wait_time="5 mins", interval="2 seconds")
def check_state(match_state, invert=0, print_string="", openbmc_host="", openbmc_username="", openbmc_password="", os_host="", os_username="", os_password="", quiet=None): r""" Check that the Open BMC machine's composite state matches the specified state. On success, this keyword returns the machine's composite state as a dictionary. Description of arguments: match_state A dictionary whose key/value pairs are "state field"/ "state value". The state value is interpreted as a regular expression. Example call from robot: ${match_state}= Create Dictionary chassis=^On$ ... bmc=^Ready$ ... boot_progress=^OSStart$ ${state}= Check State &{match_state} invert If this flag is set, this function will succeed if the states do NOT match. print_string This function will print this string to the console prior to getting the state. openbmc_host The DNS name or IP address of the BMC. This defaults to global ${OPENBMC_HOST}. openbmc_username The username to be used to login to the BMC. This defaults to global ${OPENBMC_USERNAME}. openbmc_password The password to be used to login to the BMC. This defaults to global ${OPENBMC_PASSWORD}. os_host The DNS name or IP address of the operating system. This defaults to global ${OS_HOST}. os_username The username to be used to login to the OS. This defaults to global ${OS_USERNAME}. os_password The password to be used to login to the OS. This defaults to global ${OS_PASSWORD}. quiet Indicates whether status details should be written to the console. Defaults to either global value of ${QUIET} or to 1. """ quiet = int(gp.get_var_value(quiet, 0)) grp.rprint(print_string) req_states = match_state.keys() # Initialize state. state = get_state(openbmc_host=openbmc_host, openbmc_username=openbmc_username, openbmc_password=openbmc_password, os_host=os_host, os_username=os_username, os_password=os_password, req_states=req_states, quiet=quiet) if not quiet: gp.print_var(state) match = compare_states(state, match_state) if invert and match: fail_msg = "The current state of the machine matches the match" +\ " state:\n" + gp.sprint_varx("state", state) BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) elif not invert and not match: fail_msg = "The current state of the machine does NOT match the" +\ " match state:\n" +\ gp.sprint_varx("state", state) BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) return state
def shell_cmd(command_string, quiet=None, print_output=None, show_err=1, test_mode=0, time_out=None, max_attempts=1, retry_sleep_time=5, allowed_shell_rcs=[0], ignore_err=None, return_stderr=0, fork=0): r""" Run the given command string in a shell and return a tuple consisting of the shell return code and the output. Description of argument(s): command_string The command string to be run in a shell (e.g. "ls /tmp"). quiet If set to 0, this function will print "Issuing: <cmd string>" to stdout. When the quiet argument is set to None, this function will assign a default value by searching upward in the stack for the quiet variable value. If no such value is found, quiet is set to 0. print_output If this is set, this function will print the stdout/stderr generated by the shell command to stdout. show_err If show_err is set, this function will print a standardized error report if the shell command fails (i.e. if the shell command returns a shell_rc that is not in allowed_shell_rcs). Note: Error text is only printed if ALL attempts to run the command_string fail. In other words, if the command execution is ultimately successful, initial failures are hidden. test_mode If test_mode is set, this function will not actually run the command. If print_output is also set, this function will print "(test_mode) Issuing: <cmd string>" to stdout. A caller should call shell_cmd directly if they wish to have the command string run unconditionally. They should call the t_shell_cmd wrapper (defined below) if they wish to run the command string only if the prevailing test_mode variable is set to 0. time_out A time-out value expressed in seconds. If the command string has not finished executing within <time_out> seconds, it will be halted and counted as an error. max_attempts The max number of attempts that should be made to run the command string. retry_sleep_time The number of seconds to sleep between attempts. allowed_shell_rcs A list of integers indicating which shell_rc values are not to be considered errors. ignore_err Ignore error means that a failure encountered by running the command string will not be raised as a python exception. When the ignore_err argument is set to None, this function will assign a default value by searching upward in the stack for the ignore_err variable value. If no such value is found, ignore_err is set to 1. return_stderr If return_stderr is set, this function will process the stdout and stderr streams from the shell command separately. In such a case, the tuple returned by this function will consist of three values rather than just two: rc, stdout, stderr. fork Run the command string asynchronously (i.e. don't wait for status of the child process and don't try to get stdout/stderr). """ # Assign default values to some of the arguments to this function. quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0))) print_output = int(gm.dft(print_output, not quiet)) show_err = int(show_err) global_ignore_err = gp.get_var_value(ignore_err, 1) stack_ignore_err = gp.get_stack_var('ignore_err', global_ignore_err) ignore_err = int(gm.dft(ignore_err, gm.dft(stack_ignore_err, 1))) err_msg = gv.valid_value(command_string) if err_msg != "": raise ValueError(err_msg) if not quiet: gp.print_issuing(command_string, test_mode) if test_mode: if return_stderr: return 0, "", "" else: return 0, "" # Convert each list entry to a signed value. allowed_shell_rcs = fa.source_to_object(allowed_shell_rcs) allowed_shell_rcs = [gm.to_signed(x) for x in allowed_shell_rcs] if return_stderr: stderr = subprocess.PIPE else: stderr = subprocess.STDOUT shell_rc = 0 out_buf = "" err_buf = "" # Write all output to func_history_stdout rather than directly to stdout. # This allows us to decide what to print after all attempts to run the # command string have been made. func_history_stdout will contain the # complete stdout history from the current invocation of this function. func_history_stdout = "" for attempt_num in range(1, max_attempts + 1): sub_proc = subprocess.Popen(command_string, bufsize=1, shell=True, executable='/bin/bash', stdout=subprocess.PIPE, stderr=stderr) out_buf = "" err_buf = "" # Output from this loop iteration is written to func_stdout for later # processing. func_stdout = "" if fork: break command_timed_out = False if time_out is not None: # Designate a SIGALRM handling function and set alarm. signal.signal(signal.SIGALRM, shell_cmd_timed_out) signal.alarm(time_out) try: if return_stderr: for line in sub_proc.stderr: try: err_buf += line except TypeError: line = line.decode("utf-8") err_buf += line if not print_output: continue func_stdout += line for line in sub_proc.stdout: try: out_buf += line except TypeError: line = line.decode("utf-8") out_buf += line if not print_output: continue func_stdout += line except IOError: command_timed_out = True sub_proc.communicate() shell_rc = sub_proc.returncode # Restore the original SIGALRM handler and clear the alarm. signal.signal(signal.SIGALRM, original_sigalrm_handler) signal.alarm(0) if shell_rc in allowed_shell_rcs: break err_msg = "The prior shell command failed.\n" if quiet: err_msg += gp.sprint_var(command_string) if command_timed_out: err_msg += gp.sprint_var(command_timed_out) err_msg += gp.sprint_var(time_out) err_msg += gp.sprint_varx("child_pid", sub_proc.pid) err_msg += gp.sprint_var(attempt_num) err_msg += gp.sprint_var(shell_rc, gp.hexa()) err_msg += gp.sprint_var(allowed_shell_rcs, gp.hexa()) if not print_output: if return_stderr: err_msg += "err_buf:\n" + err_buf err_msg += "out_buf:\n" + out_buf if show_err: func_stdout += gp.sprint_error_report(err_msg) func_history_stdout += func_stdout if attempt_num < max_attempts: func_history_stdout += gp.sprint_issuing("time.sleep(" + str(retry_sleep_time) + ")") time.sleep(retry_sleep_time) if shell_rc not in allowed_shell_rcs: func_stdout = func_history_stdout gp.gp_print(func_stdout) if shell_rc not in allowed_shell_rcs: if not ignore_err: if robot_env: BuiltIn().fail(err_msg) else: raise ValueError("The prior shell command failed.\n") if return_stderr: return shell_rc, out_buf, err_buf else: return shell_rc, out_buf
def execute_ssh_command(cmd_buf, open_connection_args={}, login_args={}, print_out=0, print_err=0, ignore_err=1, fork=0, quiet=None, test_mode=None): r""" Run the given command in an SSH session and return the stdout, stderr and the return code. If there is no open SSH connection, this function will connect and login. Likewise, if the caller has not yet logged in to the connection, this function will do the login. NOTE: There is special handling when open_connection_args['alias'] equals "device_connection". - A write, rather than an execute_command, is done. - Only stdout is returned (no stderr or rc). - print_err, ignore_err and fork are not supported. Description of arguments: cmd_buf The command string to be run in an SSH session. open_connection_args A dictionary of arg names and values which are legal to pass to the SSHLibrary open_connection function as parms/args. At a minimum, this should contain a 'host' entry. login_args A dictionary containing the key/value pairs which are acceptable to the SSHLibrary login function as parms/args. At a minimum, this should contain a 'username' and a 'password' entry. print_out If this is set, this function will print the stdout/stderr generated by the shell command. print_err If show_err is set, this function will print a standardized error report if the shell command returns non-zero. ignore_err Indicates that errors encountered on the sshlib.execute_command are to be ignored. fork Indicates that sshlib.start is to be used rather than sshlib.execute_command. quiet Indicates whether this function should run the pissuing() function which prints an "Issuing: <cmd string>" to stdout. This defaults to the global quiet value. test_mode If test_mode is set, this function will not actually run the command. This defaults to the global test_mode value. """ gp.lprint_executing() # Obtain default values. quiet = int(gp.get_var_value(quiet, 0)) test_mode = int(gp.get_var_value(test_mode, 0)) if not quiet: gp.pissuing(cmd_buf, test_mode) gp.lpissuing(cmd_buf, test_mode) if test_mode: return "", "", 0 global sshlib max_exec_cmd_attempts = 2 # Look for existing SSH connection. # Prepare a search connection dictionary. search_connection_args = open_connection_args.copy() # Remove keys that don't work well for searches. search_connection_args.pop("timeout", None) connection = find_connection(search_connection_args) if connection: gp.lprint_timen("Found the following existing connection:") gp.lprintn(sprint_connection(connection)) if connection.alias == "": index_or_alias = connection.index else: index_or_alias = connection.alias gp.lprint_timen("Switching to existing connection: \"" + str(index_or_alias) + "\".") sshlib.switch_connection(index_or_alias) else: gp.lprint_timen("Connecting to " + open_connection_args['host'] + ".") cix = sshlib.open_connection(**open_connection_args) try: login_ssh(login_args) except Exception as login_exception: except_type, except_value, except_traceback = sys.exc_info() rc = 1 stderr = str(except_value) stdout = "" max_exec_cmd_attempts = 0 for exec_cmd_attempt_num in range(1, max_exec_cmd_attempts + 1): gp.lprint_var(exec_cmd_attempt_num) try: if fork: sshlib.start_command(cmd_buf) else: if open_connection_args['alias'] == "device_connection": stdout = sshlib.write(cmd_buf) stderr = "" rc = 0 else: stdout, stderr, rc = \ sshlib.execute_command(cmd_buf, return_stdout=True, return_stderr=True, return_rc=True) except Exception as execute_exception: except_type, except_value, except_traceback = sys.exc_info() gp.lprint_var(except_type) gp.lprint_varx("except_value", str(except_value)) if except_type is exceptions.AssertionError and\ re.match(r"Connection not open", str(except_value)): login_ssh(login_args) # Now we must continue to next loop iteration to retry the # execute_command. continue if (except_type is paramiko.ssh_exception.SSHException and re.match(r"SSH session not active", str(except_value))) or\ (except_type is socket.error and re.match(r"\[Errno 104\] Connection reset by peer", str(except_value))): # Close and re-open a connection. # Note: close_connection() doesn't appear to get rid of the # connection. It merely closes it. Since there is a concern # about over-consumption of resources, we use # close_all_connections() which also gets rid of all # connections. gp.lprint_timen("Closing all connections.") sshlib.close_all_connections() gp.lprint_timen("Connecting to " + open_connection_args['host'] + ".") cix = sshlib.open_connection(**open_connection_args) login_ssh(login_args) continue # We do not handle any other RuntimeErrors so we will raise the # exception again. sshlib.close_all_connections() raise(execute_exception) # If we get to this point, the command was executed. break if fork: return if rc != 0 and print_err: gp.print_var(rc, 1) if not print_out: gp.print_var(stderr) gp.print_var(stdout) if print_out: gp.printn(stderr + stdout) if not ignore_err: message = gp.sprint_error("The prior SSH" + " command returned a non-zero return" + " code:\n" + gp.sprint_var(rc, 1) + stderr + "\n") BuiltIn().should_be_equal(rc, 0, message) if open_connection_args['alias'] == "device_connection": return stdout return stdout, stderr, rc
def execute_ssh_command(cmd_buf, open_connection_args={}, login_args={}, print_out=0, print_err=0, ignore_err=1, fork=0, quiet=None, test_mode=None): r""" Run the given command in an SSH session and return the stdout, stderr and the return code. If there is no open SSH connection, this function will connect and login. Likewise, if the caller has not yet logged in to the connection, this function will do the login. Description of arguments: cmd_buf The command string to be run in an SSH session. open_connection_args A dictionary of arg names and values which are legal to pass to the SSHLibrary open_connection function as parms/args. At a minimum, this should contain a 'host' entry. login_args A dictionary containing the key/value pairs which are acceptable to the SSHLibrary login function as parms/args. At a minimum, this should contain a 'username' and a 'password' entry. print_out If this is set, this function will print the stdout/stderr generated by the shell command. print_err If show_err is set, this function will print a standardized error report if the shell command returns non-zero. ignore_err Indicates that errors encountered on the sshlib.execute_command are to be ignored. fork Indicates that sshlib.start is to be used rather than sshlib.execute_command. quiet Indicates whether this function should run the pissuing() function which prints an "Issuing: <cmd string>" to stdout. This defaults to the global quiet value. test_mode If test_mode is set, this function will not actually run the command. This defaults to the global test_mode value. """ gp.dprint_executing() # Obtain default values. quiet = int(gp.get_var_value(quiet, 0)) test_mode = int(gp.get_var_value(test_mode, 0)) if not quiet: gp.pissuing(cmd_buf, test_mode) if test_mode: return "", "", 0 global sshlib # Look for existing SSH connection. # Prepare a search connection dictionary. search_connection_args = open_connection_args.copy() # Remove keys that don't work well for searches. search_connection_args.pop("timeout", None) connection = find_connection(search_connection_args) if connection: gp.dprint_timen("Found the following existing connection:") gp.dprintn(sprint_connection(connection)) if connection.alias == "": index_or_alias = connection.index else: index_or_alias = connection.alias gp.dprint_timen("Switching to existing connection: \"" + str(index_or_alias) + "\".") sshlib.switch_connection(index_or_alias) else: gp.dprint_timen("Connecting to " + open_connection_args['host'] + ".") cix = sshlib.open_connection(**open_connection_args) login_ssh(login_args) max_exec_cmd_attempts = 2 for exec_cmd_attempt_num in range(1, max_exec_cmd_attempts + 1): gp.dprint_var(exec_cmd_attempt_num) try: if fork: sshlib.start_command(cmd_buf) else: stdout, stderr, rc = sshlib.execute_command(cmd_buf, return_stdout=True, return_stderr=True, return_rc=True) except Exception as execute_exception: except_type, except_value, except_traceback = sys.exc_info() gp.dprint_var(except_type) gp.dprint_varx("except_value", str(except_value)) if except_type is exceptions.AssertionError and\ re.match(r"Connection not open", str(except_value)): login_ssh(login_args) # Now we must continue to next loop iteration to retry the # execute_command. continue if except_type is paramiko.ssh_exception.SSHException and\ re.match(r"SSH session not active", str(except_value)): # Close and re-open a connection. sshlib.close_connection() gp.dprint_timen("Connecting to " + open_connection_args['host'] + ".") cix = sshlib.open_connection(**open_connection_args) login_ssh(login_args) continue # We do not handle any other RuntimeErrors so we will raise the # exception again. raise(execute_exception) # If we get to this point, the command was executed. break if fork: return if rc != 0 and print_err: gp.print_var(rc, 1) if not print_out: gp.print_var(stderr) gp.print_var(stdout) if print_out: gp.printn(stderr + stdout) if not ignore_err: message = gp.sprint_error("The prior SSH" + " command returned a non-zero return" + " code:\n" + gp.sprint_var(rc, 1) + stderr + "\n") BuiltIn().should_be_equal(rc, 0, message) return stdout, stderr, rc
def check_state(match_state, invert=0, print_string="", openbmc_host="", openbmc_username="", openbmc_password="", os_host="", os_username="", os_password="", quiet=None): r""" Check that the Open BMC machine's composite state matches the specified state. On success, this keyword returns the machine's composite state as a dictionary. Description of argument(s): match_state A dictionary whose key/value pairs are "state field"/ "state value". The state value is interpreted as a regular expression. Example call from robot: ${match_state}= Create Dictionary chassis=^On$ ... bmc=^Ready$ ... boot_progress=^OSStart$ ${state}= Check State &{match_state} invert If this flag is set, this function will succeed if the states do NOT match. print_string This function will print this string to the console prior to getting the state. openbmc_host The DNS name or IP address of the BMC. This defaults to global ${OPENBMC_HOST}. openbmc_username The username to be used to login to the BMC. This defaults to global ${OPENBMC_USERNAME}. openbmc_password The password to be used to login to the BMC. This defaults to global ${OPENBMC_PASSWORD}. os_host The DNS name or IP address of the operating system. This defaults to global ${OS_HOST}. os_username The username to be used to login to the OS. This defaults to global ${OS_USERNAME}. os_password The password to be used to login to the OS. This defaults to global ${OS_PASSWORD}. quiet Indicates whether status details should be written to the console. Defaults to either global value of ${QUIET} or to 1. """ quiet = int(gp.get_var_value(quiet, 0)) gp.gp_print(print_string) try: match_state = return_state_constant(match_state) except TypeError: pass req_states = list(match_state.keys()) # Remove special-case match key from req_states. if expressions_key() in req_states: req_states.remove(expressions_key()) # Initialize state. state = get_state(openbmc_host=openbmc_host, openbmc_username=openbmc_username, openbmc_password=openbmc_password, os_host=os_host, os_username=os_username, os_password=os_password, req_states=req_states, quiet=quiet) if not quiet: gp.print_var(state) if exit_wait_early_message != "": # The exit_wait_early_message has been set by a signal handler so we # will exit "successfully". It is incumbent upon the calling function # (e.g. wait_state) to check/clear this variable and to fail # appropriately. return state match = compare_states(state, match_state) if invert and match: fail_msg = "The current state of the machine matches the match" +\ " state:\n" + gp.sprint_varx("state", state) BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) elif not invert and not match: fail_msg = "The current state of the machine does NOT match the" +\ " match state:\n" +\ gp.sprint_varx("state", state) BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) return state
def get_state(openbmc_host="", openbmc_username="", openbmc_password="", os_host="", os_username="", os_password="", req_states=default_req_states, quiet=None): r""" Get component states such as chassis state, bmc state, etc, put them into a dictionary and return them to the caller. Note that all substate values are strings. Description of arguments: openbmc_host The DNS name or IP address of the BMC. This defaults to global ${OPENBMC_HOST}. openbmc_username The username to be used to login to the BMC. This defaults to global ${OPENBMC_USERNAME}. openbmc_password The password to be used to login to the BMC. This defaults to global ${OPENBMC_PASSWORD}. os_host The DNS name or IP address of the operating system. This defaults to global ${OS_HOST}. os_username The username to be used to login to the OS. This defaults to global ${OS_USERNAME}. os_password The password to be used to login to the OS. This defaults to global ${OS_PASSWORD}. req_states This is a list of states whose values are being requested by the caller. quiet Indicates whether status details (e.g. curl commands) should be written to the console. Defaults to either global value of ${QUIET} or to 1. """ quiet = int(gp.get_var_value(quiet, 0)) # Set parm defaults where necessary and validate all parms. if openbmc_host == "": openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}") error_message = gv.svalid_value(openbmc_host, var_name="openbmc_host", invalid_values=[None, ""]) if error_message != "": BuiltIn().fail(gp.sprint_error(error_message)) if openbmc_username == "": openbmc_username = BuiltIn().get_variable_value("${OPENBMC_USERNAME}") error_message = gv.svalid_value(openbmc_username, var_name="openbmc_username", invalid_values=[None, ""]) if error_message != "": BuiltIn().fail(gp.sprint_error(error_message)) if openbmc_password == "": openbmc_password = BuiltIn().get_variable_value("${OPENBMC_PASSWORD}") error_message = gv.svalid_value(openbmc_password, var_name="openbmc_password", invalid_values=[None, ""]) if error_message != "": BuiltIn().fail(gp.sprint_error(error_message)) # NOTE: OS parms are optional. if os_host == "": os_host = BuiltIn().get_variable_value("${OS_HOST}") if os_host is None: os_host = "" if os_username is "": os_username = BuiltIn().get_variable_value("${OS_USERNAME}") if os_username is None: os_username = "" if os_password is "": os_password = BuiltIn().get_variable_value("${OS_PASSWORD}") if os_password is None: os_password = "" invalid_req_states = [ sub_state for sub_state in req_states if sub_state not in valid_req_states ] if len(invalid_req_states) > 0: error_message = "The following req_states are not supported:\n" +\ gp.sprint_var(invalid_req_states) BuiltIn().fail(gp.sprint_error(error_message)) # Initialize all substate values supported by this function. ping = 0 packet_loss = '' uptime = '' epoch_seconds = '' rest = '1' chassis = '' bmc = '' boot_progress = '' host = '' # Get the component states. if 'ping' in req_states: # See if the OS pings. cmd_buf = "ping -c 1 -w 2 " + openbmc_host if not quiet: gp.pissuing(cmd_buf) rc, out_buf = commands.getstatusoutput(cmd_buf) if rc == 0: ping = 1 if 'packet_loss' in req_states: # See if the OS pings. cmd_buf = "ping -c 5 -w 5 " + openbmc_host +\ " | egrep 'packet loss' | sed -re 's/.* ([0-9]+)%.*/\\1/g'" if not quiet: gp.pissuing(cmd_buf) rc, out_buf = commands.getstatusoutput(cmd_buf) if rc == 0: packet_loss = out_buf.rstrip("\n") if 'uptime' in req_states: cmd_buf = [ "BMC Execute Command", "cat /proc/uptime | cut -f 1 -d ' '", 'quiet=${1}' ] if not quiet: grp.rpissuing_keyword(cmd_buf) status, ret_values = \ BuiltIn().run_keyword_and_ignore_error(*cmd_buf) if status == "PASS": stdout, stderr, rc = ret_values if rc == 0 and stderr == "": uptime = stdout if 'epoch_seconds' in req_states: date_cmd_buf = "date -u +%s" if USE_BMC_EPOCH_TIME: cmd_buf = ["BMC Execute Command", date_cmd_buf, 'quiet=${1}'] if not quiet: grp.rpissuing_keyword(cmd_buf) status, ret_values = \ BuiltIn().run_keyword_and_ignore_error(*cmd_buf) if status == "PASS": stdout, stderr, rc = ret_values if rc == 0 and stderr == "": epoch_seconds = stdout.rstrip("\n") else: shell_rc, out_buf = gc.cmd_fnc_u(date_cmd_buf, quiet=1, print_output=0) if shell_rc == 0: epoch_seconds = out_buf.rstrip("\n") master_req_rest = ['rest', 'chassis', 'bmc', 'boot_progress', 'host'] req_rest = [ sub_state for sub_state in req_states if sub_state in master_req_rest ] need_rest = (len(req_rest) > 0) # Though we could try to determine 'rest' state on any of several calls, # for simplicity, we'll use 'chassis' to figure it out (even if the caller # hasn't explicitly asked for 'chassis'). if 'chassis' in req_states or need_rest: cmd_buf = ["Get Chassis Power State", "quiet=${" + str(quiet) + "}"] grp.rdpissuing_keyword(cmd_buf) status, ret_values = \ BuiltIn().run_keyword_and_ignore_error(*cmd_buf) if status == "PASS": chassis = ret_values chassis = re.sub(r'.*\.', "", chassis) rest = '1' else: rest = ret_values if rest == '1': if 'bmc' in req_states: if OBMC_STATES_VERSION == 0: qualifier = "utils" else: # This will not be supported much longer. qualifier = "state_manager" cmd_buf = [ qualifier + ".Get BMC State", "quiet=${" + str(quiet) + "}" ] grp.rdpissuing_keyword(cmd_buf) status, ret_values = \ BuiltIn().run_keyword_and_ignore_error(*cmd_buf) if status == "PASS": bmc = ret_values if 'boot_progress' in req_states: cmd_buf = ["Get Boot Progress", "quiet=${" + str(quiet) + "}"] grp.rdpissuing_keyword(cmd_buf) status, ret_values = \ BuiltIn().run_keyword_and_ignore_error(*cmd_buf) if status == "PASS": boot_progress = ret_values if 'host' in req_states: if OBMC_STATES_VERSION > 0: cmd_buf = ["Get Host State", "quiet=${" + str(quiet) + "}"] grp.rdpissuing_keyword(cmd_buf) status, ret_values = \ BuiltIn().run_keyword_and_ignore_error(*cmd_buf) if status == "PASS": host = ret_values # Strip everything up to the final period. host = re.sub(r'.*\.', "", host) state = DotDict() for sub_state in req_states: if sub_state.startswith("os_"): # We pass "os_" requests on to get_os_state. continue cmd_buf = "state['" + sub_state + "'] = str(" + sub_state + ")" exec(cmd_buf) if os_host == "": # The caller has not specified an os_host so as far as we're concerned, # it doesn't exist. return state os_req_states = [ sub_state for sub_state in req_states if sub_state.startswith('os_') ] if len(os_req_states) > 0: # The caller has specified an os_host and they have requested # information on os substates. # Based on the information gathered on bmc, we'll try to make a # determination of whether the os is even up. We'll pass the result # of that assessment to get_os_state to enhance performance. os_up_match = DotDict() for sub_state in master_os_up_match: if sub_state in req_states: os_up_match[sub_state] = master_os_up_match[sub_state] os_up = compare_states(state, os_up_match) os_state = get_os_state(os_host=os_host, os_username=os_username, os_password=os_password, req_states=os_req_states, os_up=os_up, quiet=quiet) # Append os_state dictionary to ours. state.update(os_state) return state
def get_os_state(os_host="", os_username="", os_password="", req_states=default_os_req_states, os_up=True, quiet=None): r""" Get component states for the operating system such as ping, login, etc, put them into a dictionary and return them to the caller. Note that all substate values are strings. Description of arguments: os_host The DNS name or IP address of the operating system. This defaults to global ${OS_HOST}. os_username The username to be used to login to the OS. This defaults to global ${OS_USERNAME}. os_password The password to be used to login to the OS. This defaults to global ${OS_PASSWORD}. req_states This is a list of states whose values are being requested by the caller. os_up If the caller knows that the os can't possibly be up, it can improve performance by passing os_up=False. This function will then simply return default values for all requested os sub states. quiet Indicates whether status details (e.g. curl commands) should be written to the console. Defaults to either global value of ${QUIET} or to 1. """ quiet = int(gp.get_var_value(quiet, 0)) # Set parm defaults where necessary and validate all parms. if os_host == "": os_host = BuiltIn().get_variable_value("${OS_HOST}") error_message = gv.svalid_value(os_host, var_name="os_host", invalid_values=[None, ""]) if error_message != "": BuiltIn().fail(gp.sprint_error(error_message)) if os_username == "": os_username = BuiltIn().get_variable_value("${OS_USERNAME}") error_message = gv.svalid_value(os_username, var_name="os_username", invalid_values=[None, ""]) if error_message != "": BuiltIn().fail(gp.sprint_error(error_message)) if os_password == "": os_password = BuiltIn().get_variable_value("${OS_PASSWORD}") error_message = gv.svalid_value(os_password, var_name="os_password", invalid_values=[None, ""]) if error_message != "": BuiltIn().fail(gp.sprint_error(error_message)) invalid_req_states = [ sub_state for sub_state in req_states if sub_state not in valid_os_req_states ] if len(invalid_req_states) > 0: error_message = "The following req_states are not supported:\n" +\ gp.sprint_var(invalid_req_states) BuiltIn().fail(gp.sprint_error(error_message)) # Initialize all substate values supported by this function. os_ping = 0 os_login = 0 os_run_cmd = 0 if os_up: if 'os_ping' in req_states: # See if the OS pings. cmd_buf = "ping -c 1 -w 2 " + os_host if not quiet: gp.pissuing(cmd_buf) rc, out_buf = commands.getstatusoutput(cmd_buf) if rc == 0: os_ping = 1 # Programming note: All attributes which do not require an ssh login # should have been processed by this point. master_req_login = ['os_login', 'os_run_cmd'] req_login = [ sub_state for sub_state in req_states if sub_state in master_req_login ] must_login = (len(req_login) > 0) if must_login: # Open SSH connection to OS. Note that this doesn't fail even when # the OS is not up. cmd_buf = ["SSHLibrary.Open Connection", os_host] if not quiet: grp.rpissuing_keyword(cmd_buf) ix = BuiltIn().run_keyword(*cmd_buf) # Login to OS. cmd_buf = ["Login", os_username, os_password] if not quiet: grp.rpissuing_keyword(cmd_buf) status, ret_values = \ BuiltIn().run_keyword_and_ignore_error(*cmd_buf) if status == "PASS": os_login = 1 else: gp.dprint_var(status) gp.dprint_var(ret_values) if os_login: if 'os_run_cmd' in req_states: # Try running a simple command (uptime) on the OS. cmd_buf = [ "Execute Command", "uptime", "return_stderr=True", "return_rc=True" ] if not quiet: grp.rpissuing_keyword(cmd_buf) # Note that in spite of its name, there are occasions # where run_keyword_and_ignore_error can fail. status, ret_values = \ BuiltIn().run_keyword_and_ignore_error(*cmd_buf) if status == "PASS": stdout, stderr, rc = ret_values if rc == 0 and stderr == "": os_run_cmd = 1 else: gp.dprint_var(status) gp.dprint_var(stdout) gp.dprint_var(stderr) gp.dprint_var(rc) else: gp.dprint_var(status) gp.dprint_var(ret_values) os_state = DotDict() for sub_state in req_states: cmd_buf = "os_state['" + sub_state + "'] = str(" + sub_state + ")" exec(cmd_buf) return os_state
def build_ipmi_ext_cmd(quiet=None): r""" Build the global IPMI_EXT_CMD variable. If global variable IPMI_EXT_CMD already has a value, this keyword will simply return without taking any action. This keyword is designed for use by keywords which use the IPMI_EXT_CMD variable (e.g. 'Run External IPMI Raw Command'). This keyword is warranted because the ipmitool program may or may not accept the -U (i.e. username) parameter depending on the version of code loaded on the BMC. This keyword will determine whether the "-U" parameter should be used and create IPMI_EXT_CMD accordingly. Furthermore, this keyword will run the command to create the 'root' IPMI username. Description of argument(s): # quiet Indicates whether this keyword should run # without any output to the console. """ ipmi_ext_cmd = BuiltIn().get_variable_value("${IPMI_EXT_CMD}", "") if ipmi_ext_cmd != "": return quiet = int(gp.get_var_value(quiet, 0)) openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}") ipmi_username = BuiltIn().get_variable_value("${IPMI_USERNAME}", "root") ipmi_password = BuiltIn().get_variable_value("${IPMI_PASSWORD}", "0penBmc") ipmi_cipher_level = BuiltIn().get_variable_value("${IPMI_CIPHER_LEVEL}", "3") old_ipmi_ext_cmd = "ipmitool -I lanplus -C " + str(ipmi_cipher_level)\ + " -P " + ipmi_password new_ipmi_ext_cmd = "ipmitool -I lanplus -C " + str(ipmi_cipher_level)\ + " -U " + ipmi_username + " -P " + ipmi_password # Use a basic ipmitool command to help us determine whether the BMC will # accept the -U parm. ipmi_cmd = "power status" ipmi_cmd_suffix = " -H " + openbmc_host + " " + ipmi_cmd print_output = 0 cmd_buf = new_ipmi_ext_cmd + ipmi_cmd_suffix new_rc, stdout = gc.shell_cmd(cmd_buf, print_output=print_output, show_err=0, ignore_err=1) gp.qprint_varx("rc", new_rc, 1) if new_rc == 0: ipmi_ext_cmd = new_ipmi_ext_cmd BuiltIn().set_global_variable("${IPMI_EXT_CMD}", ipmi_ext_cmd) return cmd_buf = old_ipmi_ext_cmd + ipmi_cmd_suffix old_rc, stdout = gc.shell_cmd(cmd_buf, print_output=print_output, show_err=0, ignore_err=1) gp.qprint_varx("rc", old_rc, 1) if old_rc == 0: ipmi_ext_cmd = old_ipmi_ext_cmd BuiltIn().set_global_variable("${IPMI_EXT_CMD}", ipmi_ext_cmd) return message = "Unable to run ipmitool, (with or without the '-U' parm)." BuiltIn().fail(message)
def execute_ssh_command(cmd_buf, open_connection_args={}, login_args={}, print_out=0, print_err=0, ignore_err=1, fork=0, quiet=None, test_mode=None, time_out=None): r""" Run the given command in an SSH session and return the stdout, stderr and the return code. If there is no open SSH connection, this function will connect and login. Likewise, if the caller has not yet logged in to the connection, this function will do the login. NOTE: There is special handling when open_connection_args['alias'] equals "device_connection". - A write, rather than an execute_command, is done. - Only stdout is returned (no stderr or rc). - print_err, ignore_err and fork are not supported. Description of arguments: cmd_buf The command string to be run in an SSH session. open_connection_args A dictionary of arg names and values which are legal to pass to the SSHLibrary open_connection function as parms/args. At a minimum, this should contain a 'host' entry. login_args A dictionary containing the key/value pairs which are acceptable to the SSHLibrary login function as parms/args. At a minimum, this should contain a 'username' and a 'password' entry. print_out If this is set, this function will print the stdout/stderr generated by the shell command. print_err If show_err is set, this function will print a standardized error report if the shell command returns non-zero. ignore_err Indicates that errors encountered on the sshlib.execute_command are to be ignored. fork Indicates that sshlib.start is to be used rather than sshlib.execute_command. quiet Indicates whether this function should run the pissuing() function which prints an "Issuing: <cmd string>" to stdout. This defaults to the global quiet value. test_mode If test_mode is set, this function will not actually run the command. This defaults to the global test_mode value. time_out The amount of time to allow for the execution of cmd_buf. A value of None means that there is no limit to how long the command may take. """ gp.lprint_executing() # Obtain default values. quiet = int(gp.get_var_value(quiet, 0)) test_mode = int(gp.get_var_value(test_mode, 0)) if not quiet: gp.pissuing(cmd_buf, test_mode) gp.lpissuing(cmd_buf, test_mode) if test_mode: return "", "", 0 global sshlib max_exec_cmd_attempts = 2 # Look for existing SSH connection. # Prepare a search connection dictionary. search_connection_args = open_connection_args.copy() # Remove keys that don't work well for searches. search_connection_args.pop("timeout", None) connection = find_connection(search_connection_args) if connection: gp.lprint_timen("Found the following existing connection:") gp.lprintn(sprint_connection(connection)) if connection.alias == "": index_or_alias = connection.index else: index_or_alias = connection.alias gp.lprint_timen("Switching to existing connection: \"" + str(index_or_alias) + "\".") sshlib.switch_connection(index_or_alias) else: gp.lprint_timen("Connecting to " + open_connection_args['host'] + ".") cix = sshlib.open_connection(**open_connection_args) try: login_ssh(login_args) except Exception: except_type, except_value, except_traceback = sys.exc_info() rc = 1 stderr = str(except_value) stdout = "" max_exec_cmd_attempts = 0 for exec_cmd_attempt_num in range(1, max_exec_cmd_attempts + 1): gp.lprint_var(exec_cmd_attempt_num) try: if fork: sshlib.start_command(cmd_buf) else: if open_connection_args['alias'] == "device_connection": stdout = sshlib.write(cmd_buf) stderr = "" rc = 0 else: stdout, stderr, rc = \ func_timer.run(sshlib.execute_command, cmd_buf, return_stdout=True, return_stderr=True, return_rc=True, time_out=time_out) except Exception: except_type, except_value, except_traceback = sys.exc_info() gp.lprint_var(except_type) gp.lprint_varx("except_value", str(except_value)) # This may be our last time through the retry loop, so setting # return variables. rc = 1 stderr = str(except_value) stdout = "" if except_type is exceptions.AssertionError and\ re.match(r"Connection not open", str(except_value)): try: login_ssh(login_args) # Now we must continue to next loop iteration to retry the # execute_command. continue except Exception: except_type, except_value, except_traceback =\ sys.exc_info() rc = 1 stderr = str(except_value) stdout = "" break if (except_type is paramiko.ssh_exception.SSHException and re.match(r"SSH session not active", str(except_value))) or\ (except_type is socket.error and re.match(r"\[Errno 104\] Connection reset by peer", str(except_value))) or\ (except_type is paramiko.ssh_exception.SSHException and re.match(r"Timeout opening channel\.", str(except_value))): # Close and re-open a connection. # Note: close_connection() doesn't appear to get rid of the # connection. It merely closes it. Since there is a concern # about over-consumption of resources, we use # close_all_connections() which also gets rid of all # connections. gp.lprint_timen("Closing all connections.") sshlib.close_all_connections() gp.lprint_timen("Connecting to " + open_connection_args['host'] + ".") cix = sshlib.open_connection(**open_connection_args) login_ssh(login_args) continue # We do not handle any other RuntimeErrors so we will raise the # exception again. sshlib.close_all_connections() gp.lprintn(traceback.format_exc()) raise (except_value) # If we get to this point, the command was executed. break if fork: return if rc != 0 and print_err: gp.print_var(rc, gp.hexa()) if not print_out: gp.print_var(stderr) gp.print_var(stdout) if print_out: gp.printn(stderr + stdout) if not ignore_err: message = gp.sprint_error("The prior SSH" + " command returned a non-zero return" + " code:\n" + gp.sprint_var(rc, gp.hexa()) + stderr + "\n") BuiltIn().should_be_equal(rc, 0, message) if open_connection_args['alias'] == "device_connection": return stdout return stdout, stderr, rc
def run_key(keyword_buf, quiet=None, test_mode=None, ignore=0): r""" Run the given keyword, return the status and the keyword return values. The advantage of using this function verses having robot simply run your keyword is the handling of parameters like quiet, test_mode and ignore. Description of arguments: keyword_buf The keyword string to be run. quiet Indicates whether this function should run the pissuing function to print 'Issuing: <keyword string>' to stdout. test_mode If test_mode is set, this function will not actually run the command. If quiet is 0, it will print a message indicating what it would have run (e.g. "Issuing: (test_mode) your command"). ignore Ignore errors from running keyword. If this is 0, this function will fail with whatever error occurred when running the keyword. Example usage from a robot script: ${status} ${ret_values}= Run Key My Keyword \ Arg1 \ Arg2 Note that to get robot to pass your command + args as a single string to this function, you must escape extra spaces with a backslash. Also note that ret_values is a python list: ret_values: ret_values[0]: value1 ret_values[1]: value2 """ # Set these vars to default values if they are None. quiet = int(gp.get_var_value(quiet, 0)) test_mode = int(gp.get_var_value(test_mode, 0)) ignore = int(ignore) # Convert the keyword_buf into a list split wherever 2 or more spaces are # found. keyword_list = keyword_buf.split(' ') # Strip spaces from each argument to make the output look clean and # uniform. keyword_list = [item.strip(' ') for item in keyword_list] if not quiet: # Join the list back into keyword_buf for the sake of output. keyword_buf = ' '.join(keyword_list) gp.pissuing(keyword_buf, test_mode) if test_mode: return 'PASS', "" try: status, ret_values = \ BuiltIn().run_keyword_and_ignore_error(*keyword_list) except Exception as my_assertion_error: status = "FAIL" ret_values = my_assertion_error.args[0] if status != 'PASS': # Output the error message to stderr. BuiltIn().log_to_console(ret_values, stream='STDERR') if not ignore: # Fail with the given error message. BuiltIn().fail(ret_values) return status, ret_values
def get_os_state(os_host="", os_username="", os_password="", req_states=default_os_req_states, os_up=True, quiet=None): r""" Get component states for the operating system such as ping, login, etc, put them into a dictionary and return them to the caller. Note that all substate values are strings. Description of argument(s): os_host The DNS name or IP address of the operating system. This defaults to global ${OS_HOST}. os_username The username to be used to login to the OS. This defaults to global ${OS_USERNAME}. os_password The password to be used to login to the OS. This defaults to global ${OS_PASSWORD}. req_states This is a list of states whose values are being requested by the caller. os_up If the caller knows that the os can't possibly be up, it can improve performance by passing os_up=False. This function will then simply return default values for all requested os sub states. quiet Indicates whether status details (e.g. curl commands) should be written to the console. Defaults to either global value of ${QUIET} or to 1. """ quiet = int(gp.get_var_value(quiet, 0)) # Set parm defaults where necessary and validate all parms. if os_host == "": os_host = BuiltIn().get_variable_value("${OS_HOST}") error_message = gv.valid_value(os_host, invalid_values=[None, ""]) if error_message != "": BuiltIn().fail(gp.sprint_error(error_message)) if os_username == "": os_username = BuiltIn().get_variable_value("${OS_USERNAME}") error_message = gv.valid_value(os_username, invalid_values=[None, ""]) if error_message != "": BuiltIn().fail(gp.sprint_error(error_message)) if os_password == "": os_password = BuiltIn().get_variable_value("${OS_PASSWORD}") error_message = gv.valid_value(os_password, invalid_values=[None, ""]) if error_message != "": BuiltIn().fail(gp.sprint_error(error_message)) invalid_req_states = [sub_state for sub_state in req_states if sub_state not in valid_os_req_states] if len(invalid_req_states) > 0: error_message = "The following req_states are not supported:\n" +\ gp.sprint_var(invalid_req_states) BuiltIn().fail(gp.sprint_error(error_message)) # Initialize all substate values supported by this function. os_ping = 0 os_login = 0 os_run_cmd = 0 if os_up: if 'os_ping' in req_states: # See if the OS pings. rc, out_buf = gc.shell_cmd("ping -c 1 -w 2 " + os_host, print_output=0, show_err=0, ignore_err=1) if rc == 0: os_ping = 1 # Programming note: All attributes which do not require an ssh login # should have been processed by this point. master_req_login = ['os_login', 'os_run_cmd'] req_login = [sub_state for sub_state in req_states if sub_state in master_req_login] must_login = (len(req_login) > 0) if must_login: output, stderr, rc = bsu.os_execute_command("uptime", quiet=quiet, ignore_err=1, time_out=20) if rc == 0: os_login = 1 os_run_cmd = 1 else: gp.dprint_vars(output, stderr) gp.dprint_vars(rc, 1) os_state = DotDict() for sub_state in req_states: cmd_buf = "os_state['" + sub_state + "'] = str(" + sub_state + ")" exec(cmd_buf) return os_state
def get_state(openbmc_host="", openbmc_username="", openbmc_password="", os_host="", os_username="", os_password="", req_states=default_req_states, quiet=None): r""" Get component states such as chassis state, bmc state, etc, put them into a dictionary and return them to the caller. Note that all substate values are strings. Note: If elapsed_boot_time is included in req_states, it is the caller's duty to call set_start_boot_seconds() in order to set global start_boot_seconds. elapsed_boot_time is the current time minus start_boot_seconds. Description of argument(s): openbmc_host The DNS name or IP address of the BMC. This defaults to global ${OPENBMC_HOST}. openbmc_username The username to be used to login to the BMC. This defaults to global ${OPENBMC_USERNAME}. openbmc_password The password to be used to login to the BMC. This defaults to global ${OPENBMC_PASSWORD}. os_host The DNS name or IP address of the operating system. This defaults to global ${OS_HOST}. os_username The username to be used to login to the OS. This defaults to global ${OS_USERNAME}. os_password The password to be used to login to the OS. This defaults to global ${OS_PASSWORD}. req_states This is a list of states whose values are being requested by the caller. quiet Indicates whether status details (e.g. curl commands) should be written to the console. Defaults to either global value of ${QUIET} or to 1. """ quiet = int(gp.get_var_value(quiet, 0)) # Set parm defaults where necessary and validate all parms. if openbmc_host == "": openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}") error_message = gv.valid_value(openbmc_host, invalid_values=[None, ""]) if error_message != "": BuiltIn().fail(gp.sprint_error(error_message)) if openbmc_username == "": openbmc_username = BuiltIn().get_variable_value("${OPENBMC_USERNAME}") error_message = gv.valid_value(openbmc_username, invalid_values=[None, ""]) if error_message != "": BuiltIn().fail(gp.sprint_error(error_message)) if openbmc_password == "": openbmc_password = BuiltIn().get_variable_value("${OPENBMC_PASSWORD}") error_message = gv.valid_value(openbmc_password, invalid_values=[None, ""]) if error_message != "": BuiltIn().fail(gp.sprint_error(error_message)) # NOTE: OS parms are optional. if os_host == "": os_host = BuiltIn().get_variable_value("${OS_HOST}") if os_host is None: os_host = "" if os_username is "": os_username = BuiltIn().get_variable_value("${OS_USERNAME}") if os_username is None: os_username = "" if os_password is "": os_password = BuiltIn().get_variable_value("${OS_PASSWORD}") if os_password is None: os_password = "" invalid_req_states = [sub_state for sub_state in req_states if sub_state not in valid_req_states] if len(invalid_req_states) > 0: error_message = "The following req_states are not supported:\n" +\ gp.sprint_var(invalid_req_states) BuiltIn().fail(gp.sprint_error(error_message)) # Initialize all substate values supported by this function. ping = 0 packet_loss = '' uptime = '' epoch_seconds = '' elapsed_boot_time = '' rest = '' chassis = '' requested_chassis = '' bmc = '' requested_bmc = '' boot_progress = '' operating_system = '' host = '' requested_host = '' attempts_left = '' # Get the component states. if 'ping' in req_states: # See if the OS pings. rc, out_buf = gc.shell_cmd("ping -c 1 -w 2 " + openbmc_host, print_output=0, show_err=0, ignore_err=1) if rc == 0: ping = 1 if 'packet_loss' in req_states: # See if the OS pings. cmd_buf = "ping -c 5 -w 5 " + openbmc_host +\ " | egrep 'packet loss' | sed -re 's/.* ([0-9]+)%.*/\\1/g'" rc, out_buf = gc.shell_cmd(cmd_buf, print_output=0, show_err=0, ignore_err=1) if rc == 0: packet_loss = out_buf.rstrip("\n") if 'uptime' in req_states: # Sometimes reading uptime results in a blank value. Call with # wait_until_keyword_succeeds to ensure a non-blank value is obtained. remote_cmd_buf = "read uptime filler 2>/dev/null < /proc/uptime" +\ " && [ ! -z \"${uptime}\" ] && echo ${uptime}" cmd_buf = ["BMC Execute Command", re.sub('\\$', '\\$', remote_cmd_buf), 'quiet=1', 'test_mode=0'] gp.qprint_issuing(cmd_buf, 0) gp.qprint_issuing(remote_cmd_buf, 0) try: stdout, stderr, rc =\ BuiltIn().wait_until_keyword_succeeds("10 sec", "0 sec", *cmd_buf) if rc == 0 and stderr == "": uptime = stdout except AssertionError as my_assertion_error: pass if 'epoch_seconds' in req_states or 'elapsed_boot_time' in req_states: date_cmd_buf = "date -u +%s" if USE_BMC_EPOCH_TIME: cmd_buf = ["BMC Execute Command", date_cmd_buf, 'quiet=${1}'] if not quiet: gp.print_issuing(cmd_buf) status, ret_values = \ BuiltIn().run_keyword_and_ignore_error(*cmd_buf) if status == "PASS": stdout, stderr, rc = ret_values if rc == 0 and stderr == "": epoch_seconds = stdout.rstrip("\n") else: shell_rc, out_buf = gc.cmd_fnc_u(date_cmd_buf, quiet=quiet, print_output=0) if shell_rc == 0: epoch_seconds = out_buf.rstrip("\n") if 'elapsed_boot_time' in req_states: global start_boot_seconds elapsed_boot_time = int(epoch_seconds) - start_boot_seconds master_req_rest = ['rest', 'host', 'requested_host', 'operating_system', 'attempts_left', 'boot_progress', 'chassis', 'requested_chassis' 'bmc' 'requested_bmc'] req_rest = [sub_state for sub_state in req_states if sub_state in master_req_rest] need_rest = (len(req_rest) > 0) state = DotDict() if need_rest: cmd_buf = ["Read Properties", SYSTEM_STATE_URI + "enumerate", "quiet=${" + str(quiet) + "}"] gp.dprint_issuing(cmd_buf) status, ret_values = \ BuiltIn().run_keyword_and_ignore_error(*cmd_buf) if status == "PASS": state['rest'] = '1' else: state['rest'] = '0' if int(state['rest']): for url_path in ret_values: for attr_name in ret_values[url_path]: # Create a state key value based on the attr_name. try: ret_values[url_path][attr_name] = \ re.sub(r'.*\.', "", ret_values[url_path][attr_name]) except TypeError: pass # Do some key name manipulations. new_attr_name = re.sub(r'^Current|(State|Transition)$', "", attr_name) new_attr_name = re.sub(r'BMC', r'Bmc', new_attr_name) new_attr_name = re.sub(r'([A-Z][a-z])', r'_\1', new_attr_name) new_attr_name = new_attr_name.lower().lstrip("_") new_attr_name = re.sub(r'power', r'chassis', new_attr_name) if new_attr_name in req_states: state[new_attr_name] = ret_values[url_path][attr_name] for sub_state in req_states: if sub_state in state: continue if sub_state.startswith("os_"): # We pass "os_" requests on to get_os_state. continue cmd_buf = "state['" + sub_state + "'] = str(" + sub_state + ")" exec(cmd_buf) if os_host == "": # The caller has not specified an os_host so as far as we're concerned, # it doesn't exist. return state os_req_states = [sub_state for sub_state in req_states if sub_state.startswith('os_')] if len(os_req_states) > 0: # The caller has specified an os_host and they have requested # information on os substates. # Based on the information gathered on bmc, we'll try to make a # determination of whether the os is even up. We'll pass the result # of that assessment to get_os_state to enhance performance. os_up_match = DotDict() for sub_state in master_os_up_match: if sub_state in req_states: os_up_match[sub_state] = master_os_up_match[sub_state] os_up = compare_states(state, os_up_match) os_state = get_os_state(os_host=os_host, os_username=os_username, os_password=os_password, req_states=os_req_states, os_up=os_up, quiet=quiet) # Append os_state dictionary to ours. state.update(os_state) return state
def rprocess_plug_in_packages(plug_in_packages_list=None, call_point="setup", shell_rc="0x00000000", stop_on_plug_in_failure=1, stop_on_non_zero_rc=0, release_type="obmc", quiet=None, debug=None, return_history=False): r""" Call the external process_plug_in_packages.py to process the plug-in packages. Return the following: rc The return code - 0 = PASS, 1 = FAIL. shell_rc The shell return code returned by process_plug_in_packages.py. failed_plug_in_name The failed plug in name (if any). Description of arguments: plug_in_packages_list A python list of plug-in directory paths. call_point The call point program to be called for each plug-in package (e.g. post_boot). This name should not include the "cp_" prefix. shell_rc The user may supply a value other than zero to indicate an acceptable non-zero return code. For example, if this value equals 0x00000200, it means that for each plug-in call point that runs, a 0x00000200 will not be counted as a failure. stop_on_plug_in_failure If this parameter is set to 1, this program will stop and return non-zero if the call point program from any plug-in directory fails. Conversely, if it is set to false, this program will run the call point program from each and every plug-in directory regardless of their return values. Typical example cases where you'd want to run all plug-in call points regardless of success or failure would be "cleanup" or "ffdc" call points. stop_on_non_zero_rc If this parm is set to 1 and a plug-in call point program returns a valid non-zero return code (see "shell_rc" parm above), this program will stop processing and return 0 (success). Since this constitutes a successful exit, this would normally be used where the caller wishes to stop processing if one of the plug-in directory call point programs returns a special value indicating that some special case has been found. An example might be in calling some kind of "check_errl" call point program. Such a call point program might return a 2 (i.e. 0x00000200) to indicate that a given error log entry was found in an "ignore" list and is therefore to be ignored. That being the case, no other "check_errl" call point program would need to be called. release_type The type of release being tested (e.g. "obmc", "op", "fips"). This influences which integrated plug-ins are selected. quiet If quiet is set to 1, this function will NOT write status messages to stdout. This will default to the global quiet program parm or to 0. debug If this parameter is set to 1, this function will print additional debug information. This is mainly to be used by the developer of this function. This will default to the global quiet program parm or to 0. return_history In addition to rc, shell_rc and failed_plug_in_name, return a list containing historical output that looks like the following: history: history[0]: #(CDT) 2018/10/30 12:25:49 - Running OBMC_Sample/cp_post_stack """ rc = 0 plug_in_packages_list = gp.get_var_value(plug_in_packages_list, []) # If there are no plug-in packages to process, return successfully. if len(plug_in_packages_list) == 0: if return_history: return 0, 0, "", [] else: return 0, 0, "" quiet = int(gp.get_var_value(quiet, 0)) debug = int(gp.get_var_value(debug, 0)) # Create string from list. plug_in_dir_paths = ':'.join(plug_in_packages_list) temp = tempfile.NamedTemporaryFile() temp_file_path = temp.name temp2 = tempfile.NamedTemporaryFile() temp_properties_file_path = temp2.name if debug: os.environ["PERF_TRACE"] = "1" debug_string = " --quiet=0" else: debug_string = "" loc_shell_rc = 0 sub_cmd_buf = "process_plug_in_packages.py" + debug_string +\ " --call_point=" + call_point + " --allow_shell_rc=" +\ str(shell_rc) + " --stop_on_plug_in_failure=" +\ str(stop_on_plug_in_failure) + " --stop_on_non_zero_rc=" +\ str(stop_on_non_zero_rc) + " " + plug_in_dir_paths if quiet: cmd_buf = sub_cmd_buf + " > " + temp_file_path + " 2>&1" else: cmd_buf = "set -o pipefail ; " + sub_cmd_buf + " 2>&1 | tee " +\ temp_file_path if debug: gp.print_issuing(cmd_buf) else: gp.print_timen("Processing " + call_point + " call point programs.") sub_proc = subprocess.Popen(cmd_buf, shell=True, executable='/bin/bash') sub_proc.communicate() proc_plug_pkg_rc = sub_proc.returncode if return_history: # Get the "Running" statements from the output. regex = " Running [^/]+/cp_" cmd_buf = "egrep '" + regex + "' " + temp_file_path _, history = gc.shell_cmd(cmd_buf, quiet=(not debug), print_output=0, show_err=0, ignore_err=1) history = [x + "\n" for x in filter(None, history.split("\n"))] else: history = [] # As process_plug_in_packages.py help text states, it will print the values of failed_plug_in_name and # shell_rc in the following format: # failed_plug_in_name: <failed plug-in value, if any> # shell_rc: <shell return code value of last call point program> # We want to obtain those values from the output. To make the task simpler, we'll start by grepping the # output for lines that might fit such a format: # A valid bash variable against the left margin followed by... # - A colon followed by... # - Zero or more spaces bash_var_regex = "[_[:alpha:]][_[:alnum:]]*" regex = "^" + bash_var_regex + ":[ ]*" cmd_buf = "egrep '" + regex + "' " + temp_file_path + " > " +\ temp_properties_file_path gp.dprint_issuing(cmd_buf) grep_rc = os.system(cmd_buf) # Next we call my_parm_file to create a properties dictionary. properties = gm.my_parm_file(temp_properties_file_path) # Finally, we access the 2 values that we need. shell_rc = int(properties.get('shell_rc', '0x0000000000000000'), 16) failed_plug_in_name = properties.get('failed_plug_in_name', '') if proc_plug_pkg_rc != 0: if quiet: os.system("cat " + temp_file_path + " >&2") if grep_rc != 0: gp.print_var(grep_rc, gp.hexa()) gp.print_var(proc_plug_pkg_rc, gp.hexa()) gp.print_timen("Re-cap of plug-in failures:") gc.cmd_fnc_u("egrep -A 1 '^failed_plug_in_name:[ ]+' " + temp_properties_file_path + " | egrep -v '^\\--'", quiet=1, show_err=0) rc = 1 if return_history: return rc, shell_rc, failed_plug_in_name, history else: return rc, shell_rc, failed_plug_in_name
def wait_state(match_state=(), wait_time="1 min", interval="1 second", invert=0, openbmc_host="", openbmc_username="", openbmc_password="", os_host="", os_username="", os_password="", quiet=None): r""" Wait for the Open BMC machine's composite state to match the specified state. On success, this keyword returns the machine's composite state as a dictionary. Description of argument(s): match_state A dictionary whose key/value pairs are "state field"/ "state value". See check_state (above) for details. This value may also be any string accepted by return_state_constant (e.g. "standby_match_state"). In such a case this function will call return_state_constant to convert it to a proper dictionary as described above. wait_time The total amount of time to wait for the desired state. This value may be expressed in Robot Framework's time format (e.g. 1 minute, 2 min 3 s, 4.5). interval The amount of time between state checks. This value may be expressed in Robot Framework's time format (e.g. 1 minute, 2 min 3 s, 4.5). invert If this flag is set, this function will for the state of the machine to cease to match the match state. openbmc_host The DNS name or IP address of the BMC. This defaults to global ${OPENBMC_HOST}. openbmc_username The username to be used to login to the BMC. This defaults to global ${OPENBMC_USERNAME}. openbmc_password The password to be used to login to the BMC. This defaults to global ${OPENBMC_PASSWORD}. os_host The DNS name or IP address of the operating system. This defaults to global ${OS_HOST}. os_username The username to be used to login to the OS. This defaults to global ${OS_USERNAME}. os_password The password to be used to login to the OS. This defaults to global ${OS_PASSWORD}. quiet Indicates whether status details should be written to the console. Defaults to either global value of ${QUIET} or to 1. """ quiet = int(gp.get_var_value(quiet, 0)) try: match_state = return_state_constant(match_state) except TypeError: pass if not quiet: if invert: alt_text = "cease to " else: alt_text = "" gp.print_timen("Checking every " + str(interval) + " for up to " + str(wait_time) + " for the state of the machine to " + alt_text + "match the state shown below.") gp.print_var(match_state) if quiet: print_string = "" else: print_string = "#" debug = int(BuiltIn().get_variable_value("${debug}", "0")) if debug: # In debug we print state so no need to print the "#". print_string = "" check_state_quiet = 1 - debug cmd_buf = ["Check State", match_state, "invert=${" + str(invert) + "}", "print_string=" + print_string, "openbmc_host=" + openbmc_host, "openbmc_username="******"openbmc_password="******"os_host=" + os_host, "os_username="******"os_password="******"quiet=${" + str(check_state_quiet) + "}"] gp.dprint_issuing(cmd_buf) try: state = BuiltIn().wait_until_keyword_succeeds(wait_time, interval, *cmd_buf) except AssertionError as my_assertion_error: gp.printn() message = my_assertion_error.args[0] BuiltIn().fail(message) if exit_wait_early_message: # The global exit_wait_early_message was set by a signal handler # indicating that we should fail. message = exit_wait_early_message # Clear the exit_wait_early_message variable for future use. set_exit_wait_early_message("") BuiltIn().fail(gp.sprint_error(message)) if not quiet: gp.printn() if invert: gp.print_timen("The states no longer match:") else: gp.print_timen("The states match:") gp.print_var(state) return state
def run_key(keyword_buf, quiet=None, test_mode=None, ignore=0): r""" Run the given keyword, return the status and the keyword return values. The advantage of using this function verses having robot simply run your keyword is the handling of parameters like quiet, test_mode and ignore. Description of arguments: keyword_buf The keyword string to be run. quiet Indicates whether this function should run the pissuing function to print 'Issuing: <keyword string>' to stdout. test_mode If test_mode is set, this function will not actually run the command. If quiet is 0, it will print a message indicating what it would have run (e.g. "Issuing: (test_mode) your command"). ignore Ignore errors from running keyword. If this is 0, this function will fail with whatever error occurred when running the keyword. Example usage from a robot script: ${status} ${ret_values}= Run Key My Keyword \ Arg1 \ Arg2 Note that to get robot to pass your command + args as a single string to this function, you must escape extra spaces with a backslash. Also note that ret_values is a python list: ret_values: ret_values[0]: value1 ret_values[1]: value2 """ # Set these vars to default values if they are None. quiet = int(gp.get_var_value(quiet, 0)) test_mode = int(gp.get_var_value(test_mode, 0)) ignore = int(ignore) # Convert the keyword_buf into a list split wherever 2 or more spaces are # found. keyword_list = keyword_buf.split(' ') # Strip spaces from each argument to make the output look clean and # uniform. keyword_list = [item.strip(' ') for item in keyword_list] if not quiet: # Join the list back into keyword_buf for the sake of output. keyword_buf = ' '.join(keyword_list) gp.pissuing(keyword_buf, test_mode) if test_mode: return 'PASS', "" try: status, ret_values = \ BuiltIn().run_keyword_and_ignore_error(*keyword_list) except Exception as my_assertion_error: status = "FAIL" ret_values = my_assertion_error.args[0] if not (status == 'PASS' or ignore): # Output the error message to stderr. BuiltIn().log_to_console(ret_values, stream='STDERR') # Fail with the given error message. BuiltIn().fail(ret_values) return status, ret_values