def which(file_path): r""" Find the full path of an executable file and return it. The PATH environment variable dictates the results of this function. Description of arguments: file_path The relative file path (e.g. "my_file" or "lib/my_file"). """ shell_rc, out_buf = gc.cmd_fnc_u("which " + file_path, quiet=1, print_output=0, show_err=0) if shell_rc != 0: error_message = "Failed to find complete path for file \"" +\ file_path + "\".\n" error_message += gp.sprint_var(shell_rc, gp.hexa()) error_message += out_buf if robot_env: BuiltIn().fail(gp.sprint_error(error_message)) else: gp.print_error_report(error_message) return False file_path = out_buf.rstrip("\n") return file_path
def shell_cmd(command_string, quiet=None, print_output=None, show_err=1, test_mode=0, time_out=None, max_attempts=1, retry_sleep_time=5, allowed_shell_rcs=[0], ignore_err=None, return_stderr=0, fork=0): r""" Run the given command string in a shell and return a tuple consisting of the shell return code and the output. Description of argument(s): command_string The command string to be run in a shell (e.g. "ls /tmp"). quiet If set to 0, this function will print "Issuing: <cmd string>" to stdout. When the quiet argument is set to None, this function will assign a default value by searching upward in the stack for the quiet variable value. If no such value is found, quiet is set to 0. print_output If this is set, this function will print the stdout/stderr generated by the shell command to stdout. show_err If show_err is set, this function will print a standardized error report if the shell command fails (i.e. if the shell command returns a shell_rc that is not in allowed_shell_rcs). Note: Error text is only printed if ALL attempts to run the command_string fail. In other words, if the command execution is ultimately successful, initial failures are hidden. test_mode If test_mode is set, this function will not actually run the command. If print_output is also set, this function will print "(test_mode) Issuing: <cmd string>" to stdout. A caller should call shell_cmd directly if they wish to have the command string run unconditionally. They should call the t_shell_cmd wrapper (defined below) if they wish to run the command string only if the prevailing test_mode variable is set to 0. time_out A time-out value expressed in seconds. If the command string has not finished executing within <time_out> seconds, it will be halted and counted as an error. max_attempts The max number of attempts that should be made to run the command string. retry_sleep_time The number of seconds to sleep between attempts. allowed_shell_rcs A list of integers indicating which shell_rc values are not to be considered errors. ignore_err Ignore error means that a failure encountered by running the command string will not be raised as a python exception. When the ignore_err argument is set to None, this function will assign a default value by searching upward in the stack for the ignore_err variable value. If no such value is found, ignore_err is set to 1. return_stderr If return_stderr is set, this function will process the stdout and stderr streams from the shell command separately. In such a case, the tuple returned by this function will consist of three values rather than just two: rc, stdout, stderr. fork Run the command string asynchronously (i.e. don't wait for status of the child process and don't try to get stdout/stderr). """ # Assign default values to some of the arguments to this function. quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0))) print_output = int(gm.dft(print_output, not quiet)) show_err = int(show_err) global_ignore_err = gp.get_var_value(ignore_err, 1) stack_ignore_err = gp.get_stack_var('ignore_err', global_ignore_err) ignore_err = int(gm.dft(ignore_err, gm.dft(stack_ignore_err, 1))) err_msg = gv.valid_value(command_string) if err_msg != "": raise ValueError(err_msg) if not quiet: gp.print_issuing(command_string, test_mode) if test_mode: if return_stderr: return 0, "", "" else: return 0, "" # Convert each list entry to a signed value. allowed_shell_rcs = fa.source_to_object(allowed_shell_rcs) allowed_shell_rcs = [gm.to_signed(x) for x in allowed_shell_rcs] if return_stderr: stderr = subprocess.PIPE else: stderr = subprocess.STDOUT shell_rc = 0 out_buf = "" err_buf = "" # Write all output to func_history_stdout rather than directly to stdout. # This allows us to decide what to print after all attempts to run the # command string have been made. func_history_stdout will contain the # complete stdout history from the current invocation of this function. func_history_stdout = "" for attempt_num in range(1, max_attempts + 1): sub_proc = subprocess.Popen(command_string, bufsize=1, shell=True, executable='/bin/bash', stdout=subprocess.PIPE, stderr=stderr) out_buf = "" err_buf = "" # Output from this loop iteration is written to func_stdout for later # processing. func_stdout = "" if fork: break command_timed_out = False if time_out is not None: # Designate a SIGALRM handling function and set alarm. signal.signal(signal.SIGALRM, shell_cmd_timed_out) signal.alarm(time_out) try: if return_stderr: for line in sub_proc.stderr: try: err_buf += line except TypeError: line = line.decode("utf-8") err_buf += line if not print_output: continue func_stdout += line for line in sub_proc.stdout: try: out_buf += line except TypeError: line = line.decode("utf-8") out_buf += line if not print_output: continue func_stdout += line except IOError: command_timed_out = True sub_proc.communicate() shell_rc = sub_proc.returncode # Restore the original SIGALRM handler and clear the alarm. signal.signal(signal.SIGALRM, original_sigalrm_handler) signal.alarm(0) if shell_rc in allowed_shell_rcs: break err_msg = "The prior shell command failed.\n" if quiet: err_msg += gp.sprint_var(command_string) if command_timed_out: err_msg += gp.sprint_var(command_timed_out) err_msg += gp.sprint_var(time_out) err_msg += gp.sprint_varx("child_pid", sub_proc.pid) err_msg += gp.sprint_var(attempt_num) err_msg += gp.sprint_var(shell_rc, gp.hexa()) err_msg += gp.sprint_var(allowed_shell_rcs, gp.hexa()) if not print_output: if return_stderr: err_msg += "err_buf:\n" + err_buf err_msg += "out_buf:\n" + out_buf if show_err: func_stdout += gp.sprint_error_report(err_msg) func_history_stdout += func_stdout if attempt_num < max_attempts: func_history_stdout += gp.sprint_issuing("time.sleep(" + str(retry_sleep_time) + ")") time.sleep(retry_sleep_time) if shell_rc not in allowed_shell_rcs: func_stdout = func_history_stdout gp.gp_print(func_stdout) if shell_rc not in allowed_shell_rcs: if not ignore_err: if robot_env: BuiltIn().fail(err_msg) else: raise ValueError("The prior shell command failed.\n") if return_stderr: return shell_rc, out_buf, err_buf else: return shell_rc, out_buf
def cmd_fnc(cmd_buf, quiet=None, test_mode=None, debug=0, print_output=1, show_err=1, return_stderr=0, ignore_err=1): r""" Run the given command in a shell and return the shell return code and the output. Description of arguments: cmd_buf The command string to be run in a shell. quiet Indicates whether this function should run the print_issuing() function which prints "Issuing: <cmd string>" to stdout. test_mode If test_mode is set, this function will not actually run the command. If print_output is set, it will print "(test_mode) Issuing: <cmd string>" to stdout. debug If debug is set, this function will print extra debug info. print_output If this is set, this function will print the stdout/stderr generated by the shell command. show_err If show_err is set, this function will print a standardized error report if the shell command returns non-zero. return_stderr If return_stderr is set, this function will process the stdout and stderr streams from the shell command separately. It will also return stderr in addition to the return code and the stdout. """ # Determine default values. quiet = int(gm.global_default(quiet, 0)) test_mode = int(gm.global_default(test_mode, 0)) if debug: gp.print_vars(cmd_buf, quiet, test_mode, debug) err_msg = gv.valid_value(cmd_buf) if err_msg != "": raise ValueError(err_msg) if not quiet: gp.pissuing(cmd_buf, test_mode) if test_mode: if return_stderr: return 0, "", "" else: return 0, "" if return_stderr: err_buf = "" stderr = subprocess.PIPE else: stderr = subprocess.STDOUT sub_proc = subprocess.Popen(cmd_buf, bufsize=1, shell=True, executable='/bin/bash', stdout=subprocess.PIPE, stderr=stderr) out_buf = "" if return_stderr: for line in sub_proc.stderr: try: err_buf += line except TypeError: line = line.decode("utf-8") err_buf += line if not print_output: continue gp.gp_print(line) for line in sub_proc.stdout: try: out_buf += line except TypeError: line = line.decode("utf-8") out_buf += line if not print_output: continue gp.gp_print(line) if print_output and not robot_env: sys.stdout.flush() sub_proc.communicate() shell_rc = sub_proc.returncode if shell_rc != 0: err_msg = "The prior shell command failed.\n" err_msg += gp.sprint_var(shell_rc, gp.hexa()) if not print_output: err_msg += "out_buf:\n" + out_buf if show_err: gp.print_error_report(err_msg) if not ignore_err: if robot_env: BuiltIn().fail(err_msg) else: raise ValueError(err_msg) if return_stderr: return shell_rc, out_buf, err_buf else: return shell_rc, out_buf
def run_boot(boot): r""" Run the specified boot. Description of arguments: boot The name of the boot test to be performed. """ global state signal.signal(signal.SIGUSR1, stop_boot_test) gp.qprint_timen("stop_boot_test is armed.") print_test_start_message(boot) plug_in_setup() rc, shell_rc, failed_plug_in_name = \ grpi.rprocess_plug_in_packages(call_point="pre_boot") if rc != 0: error_message = "Plug-in failed with non-zero return code.\n" +\ gp.sprint_var(rc, fmt=gp.hexa()) set_default_siguser1() BuiltIn().fail(gp.sprint_error(error_message)) if test_mode: # In test mode, we'll pretend the boot worked by assigning its # required end state to the default state value. state = st.strip_anchor_state(boot_table[boot]['end']) else: # Assertion: We trust that the state data was made fresh by the # caller. gp.qprintn() if boot_table[boot]['method_type'] == "keyword": rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), boot_table[boot]['method'], quiet=quiet) if boot_table[boot]['bmc_reboot']: st.wait_for_comm_cycle(int(state['epoch_seconds'])) plug_in_setup() rc, shell_rc, failed_plug_in_name = \ grpi.rprocess_plug_in_packages(call_point="post_reboot") if rc != 0: error_message = "Plug-in failed with non-zero return code.\n" error_message += gp.sprint_var(rc, fmt=gp.hexa()) set_default_siguser1() BuiltIn().fail(gp.sprint_error(error_message)) else: match_state = st.anchor_state(state) del match_state['epoch_seconds'] # Wait for the state to change in any way. st.wait_state(match_state, wait_time=state_change_timeout, interval="10 seconds", invert=1) gp.qprintn() if boot_table[boot]['end']['chassis'] == "Off": boot_timeout = power_off_timeout else: boot_timeout = power_on_timeout st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, interval="10 seconds") plug_in_setup() rc, shell_rc, failed_plug_in_name = \ grpi.rprocess_plug_in_packages(call_point="post_boot") if rc != 0: error_message = "Plug-in failed with non-zero return code.\n" +\ gp.sprint_var(rc, fmt=gp.hexa()) set_default_siguser1() BuiltIn().fail(gp.sprint_error(error_message)) # Restore original sigusr1 handler. set_default_siguser1()
def rprocess_plug_in_packages(plug_in_packages_list=None, call_point="setup", shell_rc="0x00000000", stop_on_plug_in_failure=1, stop_on_non_zero_rc=0, release_type="obmc", quiet=None, debug=None, return_history=False): r""" Call the external process_plug_in_packages.py to process the plug-in packages. Return the following: rc The return code - 0 = PASS, 1 = FAIL. shell_rc The shell return code returned by process_plug_in_packages.py. failed_plug_in_name The failed plug in name (if any). Description of arguments: plug_in_packages_list A python list of plug-in directory paths. call_point The call point program to be called for each plug-in package (e.g. post_boot). This name should not include the "cp_" prefix. shell_rc The user may supply a value other than zero to indicate an acceptable non-zero return code. For example, if this value equals 0x00000200, it means that for each plug-in call point that runs, a 0x00000200 will not be counted as a failure. stop_on_plug_in_failure If this parameter is set to 1, this program will stop and return non-zero if the call point program from any plug-in directory fails. Conversely, if it is set to false, this program will run the call point program from each and every plug-in directory regardless of their return values. Typical example cases where you'd want to run all plug-in call points regardless of success or failure would be "cleanup" or "ffdc" call points. stop_on_non_zero_rc If this parm is set to 1 and a plug-in call point program returns a valid non-zero return code (see "shell_rc" parm above), this program will stop processing and return 0 (success). Since this constitutes a successful exit, this would normally be used where the caller wishes to stop processing if one of the plug-in directory call point programs returns a special value indicating that some special case has been found. An example might be in calling some kind of "check_errl" call point program. Such a call point program might return a 2 (i.e. 0x00000200) to indicate that a given error log entry was found in an "ignore" list and is therefore to be ignored. That being the case, no other "check_errl" call point program would need to be called. release_type The type of release being tested (e.g. "obmc", "op", "fips"). This influences which integrated plug-ins are selected. quiet If quiet is set to 1, this function will NOT write status messages to stdout. This will default to the global quiet program parm or to 0. debug If this parameter is set to 1, this function will print additional debug information. This is mainly to be used by the developer of this function. This will default to the global quiet program parm or to 0. return_history In addition to rc, shell_rc and failed_plug_in_name, return a list containing historical output that looks like the following: history: history[0]: #(CDT) 2018/10/30 12:25:49 - Running OBMC_Sample/cp_post_stack """ rc = 0 plug_in_packages_list = gp.get_var_value(plug_in_packages_list, []) # If there are no plug-in packages to process, return successfully. if len(plug_in_packages_list) == 0: if return_history: return 0, 0, "", [] else: return 0, 0, "" quiet = int(gp.get_var_value(quiet, 0)) debug = int(gp.get_var_value(debug, 0)) # Create string from list. plug_in_dir_paths = ':'.join(plug_in_packages_list) temp = tempfile.NamedTemporaryFile() temp_file_path = temp.name temp2 = tempfile.NamedTemporaryFile() temp_properties_file_path = temp2.name if debug: os.environ["PERF_TRACE"] = "1" debug_string = " --quiet=0" else: debug_string = "" loc_shell_rc = 0 sub_cmd_buf = "process_plug_in_packages.py" + debug_string +\ " --call_point=" + call_point + " --allow_shell_rc=" +\ str(shell_rc) + " --stop_on_plug_in_failure=" +\ str(stop_on_plug_in_failure) + " --stop_on_non_zero_rc=" +\ str(stop_on_non_zero_rc) + " " + plug_in_dir_paths if quiet: cmd_buf = sub_cmd_buf + " > " + temp_file_path + " 2>&1" else: cmd_buf = "set -o pipefail ; " + sub_cmd_buf + " 2>&1 | tee " +\ temp_file_path if debug: gp.print_issuing(cmd_buf) else: gp.print_timen("Processing " + call_point + " call point programs.") sub_proc = subprocess.Popen(cmd_buf, shell=True, executable='/bin/bash') sub_proc.communicate() proc_plug_pkg_rc = sub_proc.returncode if return_history: # Get the "Running" statements from the output. regex = " Running [^/]+/cp_" cmd_buf = "egrep '" + regex + "' " + temp_file_path _, history = gc.shell_cmd(cmd_buf, quiet=(not debug), print_output=0, show_err=0, ignore_err=1) history = [x + "\n" for x in filter(None, history.split("\n"))] else: history = [] # As process_plug_in_packages.py help text states, it will print the values of failed_plug_in_name and # shell_rc in the following format: # failed_plug_in_name: <failed plug-in value, if any> # shell_rc: <shell return code value of last call point program> # We want to obtain those values from the output. To make the task simpler, we'll start by grepping the # output for lines that might fit such a format: # A valid bash variable against the left margin followed by... # - A colon followed by... # - Zero or more spaces bash_var_regex = "[_[:alpha:]][_[:alnum:]]*" regex = "^" + bash_var_regex + ":[ ]*" cmd_buf = "egrep '" + regex + "' " + temp_file_path + " > " +\ temp_properties_file_path gp.dprint_issuing(cmd_buf) grep_rc = os.system(cmd_buf) # Next we call my_parm_file to create a properties dictionary. properties = gm.my_parm_file(temp_properties_file_path) # Finally, we access the 2 values that we need. shell_rc = int(properties.get('shell_rc', '0x0000000000000000'), 16) failed_plug_in_name = properties.get('failed_plug_in_name', '') if proc_plug_pkg_rc != 0: if quiet: os.system("cat " + temp_file_path + " >&2") if grep_rc != 0: gp.print_var(grep_rc, gp.hexa()) gp.print_var(proc_plug_pkg_rc, gp.hexa()) gp.print_timen("Re-cap of plug-in failures:") gc.cmd_fnc_u("egrep -A 1 '^failed_plug_in_name:[ ]+' " + temp_properties_file_path + " | egrep -v '^\\--'", quiet=1, show_err=0) rc = 1 if return_history: return rc, shell_rc, failed_plug_in_name, history else: return rc, shell_rc, failed_plug_in_name
def robot_cmd_fnc(robot_cmd_buf, robot_jail=os.environ.get('ROBOT_JAIL', ''), quiet=None, test_mode=0): r""" Run the robot command string. This function will set the various PATH variables correctly so that you are running the proper version of all imported files, etc. Description of argument(s): robot_cmd_buf The complete robot command string. robot_jail Indicates that this is to run in "robot jail" meaning without visibility to any apolloxxx import files, programs, etc. test_mode If test_mode is set, this function will not actually run the command. """ quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0))) gv.valid_value(robot_cmd_buf) # Set global variables to aid in cleanup with process_robot_output_files. global gcr_last_robot_cmd_buf global gcr_last_robot_rc gcr_last_robot_cmd_buf = robot_cmd_buf # Get globals set by init_robot_test_base_dir_path(). module = sys.modules["__main__"] try: ROBOT_TEST_BASE_DIR_PATH = getattr(module, "ROBOT_TEST_BASE_DIR_PATH") except NameError: init_robot_test_base_dir_path() ROBOT_TEST_BASE_DIR_PATH = getattr(module, "ROBOT_TEST_BASE_DIR_PATH") ROBOT_TEST_RUNNING_FROM_SB = gm.get_mod_global( "ROBOT_TEST_RUNNING_FROM_SB") OPENBMCTOOL_DIR_PATH = gm.get_mod_global("OPENBMCTOOL_DIR_PATH") if robot_jail == "": if ROBOT_TEST_RUNNING_FROM_SB: robot_jail = 0 else: robot_jail = 1 robot_jail = int(robot_jail) ROBOT_JAIL = os.environ.get('ROBOT_JAIL', '') gp.dprint_vars(ROBOT_TEST_BASE_DIR_PATH, ROBOT_TEST_RUNNING_FROM_SB, ROBOT_JAIL, robot_jail) # Save PATH and PYTHONPATH to be restored later. os.environ["SAVED_PYTHONPATH"] = os.environ.get("PYTHONPATH", "") os.environ["SAVED_PATH"] = os.environ.get("PATH", "") if robot_jail: # Make sure required programs like python and robot can be found in the new restricted PATH. required_programs = "python robot" # It is expected that there will be a "python" program in the tool base bin path which is really a # link to select_version. Ditto for "robot". Call each with the --print_only option to get the # paths to the "real" programs. cmd_buf = "for program in " + required_programs \ + " ; do dirname $(${program} --print_only) ; done 2>/dev/null" rc, out_buf = gc.shell_cmd(cmd_buf, quiet=1, print_output=0) PYTHONPATH = ROBOT_TEST_BASE_DIR_PATH + "lib" NEW_PATH_LIST = [ROBOT_TEST_BASE_DIR_PATH + "bin"] NEW_PATH_LIST.extend(list(set(out_buf.rstrip("\n").split("\n")))) NEW_PATH_LIST.extend([ "/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin", OPENBMCTOOL_DIR_PATH.rstrip('/') ]) PATH = ":".join(NEW_PATH_LIST) else: PYTHONPATH = os.environ.get('PYTHONPATH', '') + ":" +\ ROBOT_TEST_BASE_DIR_PATH + "lib" PATH = os.environ.get('PATH', '') + ":" + ROBOT_TEST_BASE_DIR_PATH +\ "bin" + ":" + OPENBMCTOOL_DIR_PATH.rstrip('/') os.environ['PYTHONPATH'] = PYTHONPATH os.environ['PATH'] = PATH gp.dprint_vars(PATH, PYTHONPATH) os.environ['FFDC_DIR_PATH_STYLE'] = os.environ.get('FFDC_DIR_PATH_STYLE', '1') gp.qpissuing(robot_cmd_buf, test_mode) if test_mode: os.environ["PATH"] = os.environ.get("SAVED_PATH", "") os.environ["PYTHONPATH"] = os.environ.get("SAVED_PYTHONPATH", "") return True if quiet: DEVNULL = open(os.devnull, 'wb') stdout = DEVNULL else: stdout = None sub_proc = subprocess.Popen(robot_cmd_buf, stdout=stdout, shell=True) sub_proc.communicate() shell_rc = sub_proc.returncode os.environ["PATH"] = os.environ.get("SAVED_PATH", "") os.environ["PYTHONPATH"] = os.environ.get("SAVED_PYTHONPATH", "") gcr_last_robot_rc = shell_rc process_robot_output_files() if shell_rc != 0: gp.print_var(shell_rc, gp.hexa()) return False return True
def process_robot_output_files(robot_cmd_buf=None, robot_rc=None, gzip=None): r""" Process robot output files which can involve several operations: - If the files are in a temporary location, using SAVE_STATUS_POLICY to decide whether to move them to a permanent location or to delete them. - Gzipping them. Description of argument(s): robot_cmd_buf The complete command string used to invoke robot. robot_rc The return code from running the robot command string. gzip Indicates whether robot-generated output should be gzipped. """ robot_cmd_buf = gm.dft(robot_cmd_buf, gcr_last_robot_cmd_buf) robot_rc = gm.dft(robot_rc, gcr_last_robot_rc) gzip = gm.dft(gzip, int(os.environ.get("GZIP_ROBOT", "1"))) if robot_cmd_buf == "": # This can legitimately occur if this function is called from an exit_function without the program # having ever run robot_cmd_fnc. return SAVE_STATUS_POLICY = os.environ.get("SAVE_STATUS_POLICY", "ALWAYS") gp.qprint_vars(SAVE_STATUS_POLICY) # When SAVE_STATUS_POLICY is "NEVER" robot output files don't even get generated. if SAVE_STATUS_POLICY == "NEVER": return # Compose file_list based on robot command buffer passed in. robot_cmd_buf_dict = gc.parse_command_string(robot_cmd_buf) outputdir = robot_cmd_buf_dict['outputdir'] outputdir = gm.add_trailing_slash(outputdir) file_list = outputdir + robot_cmd_buf_dict['output'] + " " + outputdir\ + robot_cmd_buf_dict['log'] + " " + outputdir\ + robot_cmd_buf_dict['report'] # Double checking that files are present. shell_rc, out_buf = gc.shell_cmd("ls -1 " + file_list + " 2>/dev/null", show_err=0) file_list = re.sub("\n", " ", out_buf.rstrip("\n")) if file_list == "": gp.qprint_timen("No robot output files were found in " + outputdir + ".") return gp.qprint_var(robot_rc, gp.hexa()) if SAVE_STATUS_POLICY == "FAIL" and robot_rc == 0: gp.qprint_timen("The call to robot produced no failures." + " Deleting robot output files.") gc.shell_cmd("rm -rf " + file_list) return if gzip: gc.shell_cmd("gzip -f " + file_list) # Update the values in file_list. file_list = re.sub(" ", ".gz ", file_list) + ".gz" # It TMP_ROBOT_DIR_PATH is set, it means the caller wanted the robot output initially directed to # TMP_ROBOT_DIR_PATH but later moved to FFDC_DIR_PATH. Otherwise, we're done. if os.environ.get("TMP_ROBOT_DIR_PATH", "") is "": return # We're directing these to the FFDC dir path so that they'll be subjected to FFDC cleanup. target_dir_path = os.environ.get("FFDC_DIR_PATH", os.environ.get("HOME", ".") + "/ffdc") target_dir_path = gm.add_trailing_slash(target_dir_path) targ_file_list = [ re.sub(".*/", target_dir_path, x) for x in file_list.split(" ") ] gc.shell_cmd("mv " + file_list + " " + target_dir_path + " >/dev/null", time_out=600) gp.qprint_timen("New robot log file locations:") gp.qprintn('\n'.join(targ_file_list))
def shell_cmd(command_string, quiet=None, print_output=None, show_err=1, test_mode=0, time_out=None, max_attempts=1, retry_sleep_time=5, valid_rcs=[0], ignore_err=None, return_stderr=0, fork=0, error_regexes=None): r""" Run the given command string in a shell and return a tuple consisting of the shell return code and the output. Description of argument(s): command_string The command string to be run in a shell (e.g. "ls /tmp"). quiet If set to 0, this function will print "Issuing: <cmd string>" to stdout. When the quiet argument is set to None, this function will assign a default value by searching upward in the stack for the quiet variable value. If no such value is found, quiet is set to 0. print_output If this is set, this function will print the stdout/stderr generated by the shell command to stdout. show_err If show_err is set, this function will print a standardized error report if the shell command fails (i.e. if the shell command returns a shell_rc that is not in valid_rcs). Note: Error text is only printed if ALL attempts to run the command_string fail. In other words, if the command execution is ultimately successful, initial failures are hidden. test_mode If test_mode is set, this function will not actually run the command. If print_output is also set, this function will print "(test_mode) Issuing: <cmd string>" to stdout. A caller should call shell_cmd directly if they wish to have the command string run unconditionally. They should call the t_shell_cmd wrapper (defined below) if they wish to run the command string only if the prevailing test_mode variable is set to 0. time_out A time-out value expressed in seconds. If the command string has not finished executing within <time_out> seconds, it will be halted and counted as an error. max_attempts The max number of attempts that should be made to run the command string. retry_sleep_time The number of seconds to sleep between attempts. valid_rcs A list of integers indicating which shell_rc values are not to be considered errors. ignore_err Ignore error means that a failure encountered by running the command string will not be raised as a python exception. When the ignore_err argument is set to None, this function will assign a default value by searching upward in the stack for the ignore_err variable value. If no such value is found, ignore_err is set to 1. return_stderr If return_stderr is set, this function will process the stdout and stderr streams from the shell command separately. In such a case, the tuple returned by this function will consist of three values rather than just two: rc, stdout, stderr. fork Run the command string asynchronously (i.e. don't wait for status of the child process and don't try to get stdout/stderr) and return the Popen object created by the subprocess.popen() function. See the kill_cmd function for details on how to process the popen object. error_regexes A list of regular expressions to be used to identify errors in the command output. If there is a match for any of these regular expressions, the command will be considered a failure and the shell_rc will be set to -1. For example, if error_regexes = ['ERROR:'] and the command output contains 'ERROR: Unrecognized option', it will be counted as an error even if the command returned 0. This is useful when running commands that do not always return non-zero on error. """ err_msg = gv.valid_value(command_string) if err_msg: raise ValueError(err_msg) # Assign default values to some of the arguments to this function. quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0))) print_output = int(gm.dft(print_output, not quiet)) show_err = int(show_err) ignore_err = int(gm.dft(ignore_err, gp.get_stack_var('ignore_err', 1))) gp.qprint_issuing(command_string, test_mode) if test_mode: return (0, "", "") if return_stderr else (0, "") # Convert a string python dictionary definition to a dictionary. valid_rcs = fa.source_to_object(valid_rcs) # Convert each list entry to a signed value. valid_rcs = [gm.to_signed(x) for x in valid_rcs] stderr = subprocess.PIPE if return_stderr else subprocess.STDOUT # Write all output to func_out_history_buf rather than directly to stdout. This allows us to decide # what to print after all attempts to run the command string have been made. func_out_history_buf will # contain the complete history from the current invocation of this function. global command_timed_out command_timed_out = False func_out_history_buf = "" for attempt_num in range(1, max_attempts + 1): sub_proc = subprocess.Popen(command_string, preexec_fn=os.setsid, bufsize=1, shell=True, universal_newlines=True, executable='/bin/bash', stdout=subprocess.PIPE, stderr=stderr) if fork: return sub_proc if time_out: command_timed_out = False # Designate a SIGALRM handling function and set alarm. signal.signal(signal.SIGALRM, shell_cmd_timed_out) signal.alarm(time_out) try: stdout_buf, stderr_buf = sub_proc.communicate() except IOError: command_timed_out = True # Restore the original SIGALRM handler and clear the alarm. signal.signal(signal.SIGALRM, original_sigalrm_handler) signal.alarm(0) # Output from this loop iteration is written to func_out_buf for later processing. This can include # stdout, stderr and our own error messages. func_out_buf = "" if print_output: if return_stderr: func_out_buf += stderr_buf func_out_buf += stdout_buf shell_rc = sub_proc.returncode if shell_rc in valid_rcs: # Check output for text indicating there is an error. if error_regexes and re.match('|'.join(error_regexes), stdout_buf): shell_rc = -1 else: break err_msg = "The prior shell command failed.\n" err_msg += gp.sprint_var(attempt_num) err_msg += gp.sprint_vars(command_string, command_timed_out, time_out) err_msg += gp.sprint_varx("child_pid", sub_proc.pid) err_msg += gp.sprint_vars(shell_rc, valid_rcs, fmt=gp.hexa()) if error_regexes: err_msg += gp.sprint_vars(error_regexes) if not print_output: if return_stderr: err_msg += "stderr_buf:\n" + stderr_buf err_msg += "stdout_buf:\n" + stdout_buf if show_err: func_out_buf += gp.sprint_error_report(err_msg) if attempt_num < max_attempts: cmd_buf = "time.sleep(" + str(retry_sleep_time) + ")" if show_err: func_out_buf += gp.sprint_issuing(cmd_buf) exec(cmd_buf) func_out_history_buf += func_out_buf if shell_rc in valid_rcs: gp.gp_print(func_out_buf) else: if show_err: gp.gp_print(func_out_history_buf, stream='stderr') else: # There is no error information to show so just print output from last loop iteration. gp.gp_print(func_out_buf) if not ignore_err: # If the caller has already asked to show error info, avoid repeating that in the failure message. err_msg = "The prior shell command failed.\n" if show_err \ else err_msg if robot_env: BuiltIn().fail(err_msg) else: raise ValueError(err_msg) return (shell_rc, stdout_buf, stderr_buf) if return_stderr \ else (shell_rc, stdout_buf)
def execute_ssh_command(cmd_buf, open_connection_args={}, login_args={}, print_out=0, print_err=0, ignore_err=1, fork=0, quiet=None, test_mode=None, time_out=None): r""" Run the given command in an SSH session and return the stdout, stderr and the return code. If there is no open SSH connection, this function will connect and login. Likewise, if the caller has not yet logged in to the connection, this function will do the login. NOTE: There is special handling when open_connection_args['alias'] equals "device_connection". - A write, rather than an execute_command, is done. - Only stdout is returned (no stderr or rc). - print_err, ignore_err and fork are not supported. Description of arguments: cmd_buf The command string to be run in an SSH session. open_connection_args A dictionary of arg names and values which are legal to pass to the SSHLibrary open_connection function as parms/args. At a minimum, this should contain a 'host' entry. login_args A dictionary containing the key/value pairs which are acceptable to the SSHLibrary login function as parms/args. At a minimum, this should contain a 'username' and a 'password' entry. print_out If this is set, this function will print the stdout/stderr generated by the shell command. print_err If show_err is set, this function will print a standardized error report if the shell command returns non-zero. ignore_err Indicates that errors encountered on the sshlib.execute_command are to be ignored. fork Indicates that sshlib.start is to be used rather than sshlib.execute_command. quiet Indicates whether this function should run the pissuing() function which prints an "Issuing: <cmd string>" to stdout. This defaults to the global quiet value. test_mode If test_mode is set, this function will not actually run the command. This defaults to the global test_mode value. time_out The amount of time to allow for the execution of cmd_buf. A value of None means that there is no limit to how long the command may take. """ gp.lprint_executing() # Obtain default values. quiet = int(gp.get_var_value(quiet, 0)) test_mode = int(gp.get_var_value(test_mode, 0)) if not quiet: gp.pissuing(cmd_buf, test_mode) gp.lpissuing(cmd_buf, test_mode) if test_mode: return "", "", 0 global sshlib max_exec_cmd_attempts = 2 # Look for existing SSH connection. # Prepare a search connection dictionary. search_connection_args = open_connection_args.copy() # Remove keys that don't work well for searches. search_connection_args.pop("timeout", None) connection = find_connection(search_connection_args) if connection: gp.lprint_timen("Found the following existing connection:") gp.lprintn(sprint_connection(connection)) if connection.alias == "": index_or_alias = connection.index else: index_or_alias = connection.alias gp.lprint_timen("Switching to existing connection: \"" + str(index_or_alias) + "\".") sshlib.switch_connection(index_or_alias) else: gp.lprint_timen("Connecting to " + open_connection_args['host'] + ".") cix = sshlib.open_connection(**open_connection_args) try: login_ssh(login_args) except Exception: except_type, except_value, except_traceback = sys.exc_info() rc = 1 stderr = str(except_value) stdout = "" max_exec_cmd_attempts = 0 for exec_cmd_attempt_num in range(1, max_exec_cmd_attempts + 1): gp.lprint_var(exec_cmd_attempt_num) try: if fork: sshlib.start_command(cmd_buf) else: if open_connection_args['alias'] == "device_connection": stdout = sshlib.write(cmd_buf) stderr = "" rc = 0 else: stdout, stderr, rc = \ func_timer.run(sshlib.execute_command, cmd_buf, return_stdout=True, return_stderr=True, return_rc=True, time_out=time_out) except Exception: except_type, except_value, except_traceback = sys.exc_info() gp.lprint_var(except_type) gp.lprint_varx("except_value", str(except_value)) # This may be our last time through the retry loop, so setting # return variables. rc = 1 stderr = str(except_value) stdout = "" if except_type is exceptions.AssertionError and\ re.match(r"Connection not open", str(except_value)): try: login_ssh(login_args) # Now we must continue to next loop iteration to retry the # execute_command. continue except Exception: except_type, except_value, except_traceback =\ sys.exc_info() rc = 1 stderr = str(except_value) stdout = "" break if (except_type is paramiko.ssh_exception.SSHException and re.match(r"SSH session not active", str(except_value))) or\ (except_type is socket.error and re.match(r"\[Errno 104\] Connection reset by peer", str(except_value))) or\ (except_type is paramiko.ssh_exception.SSHException and re.match(r"Timeout opening channel\.", str(except_value))): # Close and re-open a connection. # Note: close_connection() doesn't appear to get rid of the # connection. It merely closes it. Since there is a concern # about over-consumption of resources, we use # close_all_connections() which also gets rid of all # connections. gp.lprint_timen("Closing all connections.") sshlib.close_all_connections() gp.lprint_timen("Connecting to " + open_connection_args['host'] + ".") cix = sshlib.open_connection(**open_connection_args) login_ssh(login_args) continue # We do not handle any other RuntimeErrors so we will raise the # exception again. sshlib.close_all_connections() gp.lprintn(traceback.format_exc()) raise (except_value) # If we get to this point, the command was executed. break if fork: return if rc != 0 and print_err: gp.print_var(rc, gp.hexa()) if not print_out: gp.print_var(stderr) gp.print_var(stdout) if print_out: gp.printn(stderr + stdout) if not ignore_err: message = gp.sprint_error("The prior SSH" + " command returned a non-zero return" + " code:\n" + gp.sprint_var(rc, gp.hexa()) + stderr + "\n") BuiltIn().should_be_equal(rc, 0, message) if open_connection_args['alias'] == "device_connection": return stdout return stdout, stderr, rc