def print_report(self, header_footer="\n", quiet=None): r""" Print the formatted boot_resuls_table to the console. Description of argument(s): See sprint_report for details. quiet Only print if this value is 0. This function will search upward in the stack to get the default value. """ quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0))) gp.qprint(self.sprint_report(header_footer))
def chdir(path, quiet=None): r""" Call os.chdir with the caller's arguments. This function offers this advantage over the base os.chdir function: - It will print an "Issuing: os.chdir" message. Description of argument(s): path The path of the directory to change to. quiet Indicates whether this function should run the print_issuing() function. """ quiet = int(dft(quiet, gp.get_stack_var('quiet', 0))) gp.qprint_issuing("os.chdir('" + path + "')") os.chdir(path)
def shell_cmd_timed_out(signal_number, frame): r""" Handle an alarm signal generated during the shell_cmd function. """ gp.dprint_executing() # Get subprocess pid from shell_cmd's call stack. sub_proc = gp.get_stack_var('sub_proc', 0) pid = sub_proc.pid # Terminate the child process. os.kill(pid, signal.SIGTERM) # Restore the original SIGALRM handler. signal.signal(signal.SIGALRM, original_sigalrm_handler) return
def rmtree(path, ignore_errors=False, onerror=None, quiet=None): r""" Call shutil.rmtree with the caller's arguments. This function offers this advantage over the base function: - It will print an "Issuing: shutil.rmtree" message. Description of argument(s): (All parms are passed directly to shutil.rmtree. See its prolog for details) quiet Indicates whether this function should run the print_issuing() function. """ quiet = int(dft(quiet, gp.get_stack_var('quiet', 0))) print_string = gp.sprint_executing(max_width=2000) print_string = re.sub(r"Executing: ", "Issuing: shutil.", print_string.rstrip("\n")) gp.qprintn(re.sub(r", quiet[ ]?=.*", ")", print_string)) shutil.rmtree(path, ignore_errors, onerror)
def print_boot_history(boot_history, quiet=None): r""" Print the last ten boots done with their time stamps. Description of argument(s): quiet Only print if this value is 0. This function will search upward in the stack to get the default value. """ quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0))) # indent 0, 90 chars wide, linefeed, char is "=" gp.qprint_dashes(0, 90) gp.qprintn("Last 10 boots:\n") for boot_entry in boot_history: gp.qprint(boot_entry) gp.qprint_dashes(0, 90)
def t_shell_cmd(command_string, **kwargs): r""" Search upward in the the call stack to obtain the test_mode argument, add it to kwargs and then call shell_cmd and return the result. See shell_cmd prolog for details on all arguments. """ if 'test_mode' in kwargs: error_message = "Programmer error - test_mode is not a valid" +\ " argument to this function." gp.print_error_report(error_message) exit(1) test_mode = int(gp.get_stack_var('test_mode', 0)) kwargs['test_mode'] = test_mode return shell_cmd(command_string, **kwargs)
def makedirs(path, mode=0o777, quiet=None): r""" Call os.makedirs with the caller's arguments. This function offers 2 advantages over the base os.makedirs function: 1) It will not fail if the directory already exists. 2) It will print an "Issuing: os.makedirs" message. Description of argument(s): path The path containing the directories to be created. mode The mode or permissions to be granted to the created directories. quiet Indicates whether this function should run the print_issuing() function. """ quiet = int(dft(quiet, gp.get_stack_var('quiet', 0))) gp.qprint_issuing("os.makedirs('" + path + "', mode=" + oct(mode) + ")") try: os.makedirs(path, mode) except OSError: pass
def shell_cmd(command_string, quiet=None, print_output=None, show_err=1, test_mode=0, time_out=None, max_attempts=1, retry_sleep_time=5, allowed_shell_rcs=[0], ignore_err=None, return_stderr=0, fork=0): r""" Run the given command string in a shell and return a tuple consisting of the shell return code and the output. Description of argument(s): command_string The command string to be run in a shell (e.g. "ls /tmp"). quiet If set to 0, this function will print "Issuing: <cmd string>" to stdout. When the quiet argument is set to None, this function will assign a default value by searching upward in the stack for the quiet variable value. If no such value is found, quiet is set to 0. print_output If this is set, this function will print the stdout/stderr generated by the shell command to stdout. show_err If show_err is set, this function will print a standardized error report if the shell command fails (i.e. if the shell command returns a shell_rc that is not in allowed_shell_rcs). Note: Error text is only printed if ALL attempts to run the command_string fail. In other words, if the command execution is ultimately successful, initial failures are hidden. test_mode If test_mode is set, this function will not actually run the command. If print_output is also set, this function will print "(test_mode) Issuing: <cmd string>" to stdout. A caller should call shell_cmd directly if they wish to have the command string run unconditionally. They should call the t_shell_cmd wrapper (defined below) if they wish to run the command string only if the prevailing test_mode variable is set to 0. time_out A time-out value expressed in seconds. If the command string has not finished executing within <time_out> seconds, it will be halted and counted as an error. max_attempts The max number of attempts that should be made to run the command string. retry_sleep_time The number of seconds to sleep between attempts. allowed_shell_rcs A list of integers indicating which shell_rc values are not to be considered errors. ignore_err Ignore error means that a failure encountered by running the command string will not be raised as a python exception. When the ignore_err argument is set to None, this function will assign a default value by searching upward in the stack for the ignore_err variable value. If no such value is found, ignore_err is set to 1. return_stderr If return_stderr is set, this function will process the stdout and stderr streams from the shell command separately. In such a case, the tuple returned by this function will consist of three values rather than just two: rc, stdout, stderr. fork Run the command string asynchronously (i.e. don't wait for status of the child process and don't try to get stdout/stderr). """ # Assign default values to some of the arguments to this function. quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0))) print_output = int(gm.dft(print_output, not quiet)) show_err = int(show_err) global_ignore_err = gp.get_var_value(ignore_err, 1) stack_ignore_err = gp.get_stack_var('ignore_err', global_ignore_err) ignore_err = int(gm.dft(ignore_err, gm.dft(stack_ignore_err, 1))) err_msg = gv.valid_value(command_string) if err_msg != "": raise ValueError(err_msg) if not quiet: gp.print_issuing(command_string, test_mode) if test_mode: if return_stderr: return 0, "", "" else: return 0, "" # Convert each list entry to a signed value. allowed_shell_rcs = fa.source_to_object(allowed_shell_rcs) allowed_shell_rcs = [gm.to_signed(x) for x in allowed_shell_rcs] if return_stderr: stderr = subprocess.PIPE else: stderr = subprocess.STDOUT shell_rc = 0 out_buf = "" err_buf = "" # Write all output to func_history_stdout rather than directly to stdout. # This allows us to decide what to print after all attempts to run the # command string have been made. func_history_stdout will contain the # complete stdout history from the current invocation of this function. func_history_stdout = "" for attempt_num in range(1, max_attempts + 1): sub_proc = subprocess.Popen(command_string, bufsize=1, shell=True, executable='/bin/bash', stdout=subprocess.PIPE, stderr=stderr) out_buf = "" err_buf = "" # Output from this loop iteration is written to func_stdout for later # processing. func_stdout = "" if fork: break command_timed_out = False if time_out is not None: # Designate a SIGALRM handling function and set alarm. signal.signal(signal.SIGALRM, shell_cmd_timed_out) signal.alarm(time_out) try: if return_stderr: for line in sub_proc.stderr: try: err_buf += line except TypeError: line = line.decode("utf-8") err_buf += line if not print_output: continue func_stdout += line for line in sub_proc.stdout: try: out_buf += line except TypeError: line = line.decode("utf-8") out_buf += line if not print_output: continue func_stdout += line except IOError: command_timed_out = True sub_proc.communicate() shell_rc = sub_proc.returncode # Restore the original SIGALRM handler and clear the alarm. signal.signal(signal.SIGALRM, original_sigalrm_handler) signal.alarm(0) if shell_rc in allowed_shell_rcs: break err_msg = "The prior shell command failed.\n" if quiet: err_msg += gp.sprint_var(command_string) if command_timed_out: err_msg += gp.sprint_var(command_timed_out) err_msg += gp.sprint_var(time_out) err_msg += gp.sprint_varx("child_pid", sub_proc.pid) err_msg += gp.sprint_var(attempt_num) err_msg += gp.sprint_var(shell_rc, gp.hexa()) err_msg += gp.sprint_var(allowed_shell_rcs, gp.hexa()) if not print_output: if return_stderr: err_msg += "err_buf:\n" + err_buf err_msg += "out_buf:\n" + out_buf if show_err: func_stdout += gp.sprint_error_report(err_msg) func_history_stdout += func_stdout if attempt_num < max_attempts: func_history_stdout += gp.sprint_issuing("time.sleep(" + str(retry_sleep_time) + ")") time.sleep(retry_sleep_time) if shell_rc not in allowed_shell_rcs: func_stdout = func_history_stdout gp.gp_print(func_stdout) if shell_rc not in allowed_shell_rcs: if not ignore_err: if robot_env: BuiltIn().fail(err_msg) else: raise ValueError("The prior shell command failed.\n") if return_stderr: return shell_rc, out_buf, err_buf else: return shell_rc, out_buf
def robot_cmd_fnc(robot_cmd_buf, robot_jail=os.environ.get('ROBOT_JAIL', ''), quiet=None, test_mode=0): r""" Run the robot command string. This function will set the various PATH variables correctly so that you are running the proper version of all imported files, etc. Description of argument(s): robot_cmd_buf The complete robot command string. robot_jail Indicates that this is to run in "robot jail" meaning without visibility to any apolloxxx import files, programs, etc. test_mode If test_mode is set, this function will not actually run the command. """ quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0))) gv.valid_value(robot_cmd_buf) # Set global variables to aid in cleanup with process_robot_output_files. global gcr_last_robot_cmd_buf global gcr_last_robot_rc gcr_last_robot_cmd_buf = robot_cmd_buf # Get globals set by init_robot_test_base_dir_path(). module = sys.modules["__main__"] try: ROBOT_TEST_BASE_DIR_PATH = getattr(module, "ROBOT_TEST_BASE_DIR_PATH") except NameError: init_robot_test_base_dir_path() ROBOT_TEST_BASE_DIR_PATH = getattr(module, "ROBOT_TEST_BASE_DIR_PATH") ROBOT_TEST_RUNNING_FROM_SB = gm.get_mod_global( "ROBOT_TEST_RUNNING_FROM_SB") OPENBMCTOOL_DIR_PATH = gm.get_mod_global("OPENBMCTOOL_DIR_PATH") if robot_jail == "": if ROBOT_TEST_RUNNING_FROM_SB: robot_jail = 0 else: robot_jail = 1 robot_jail = int(robot_jail) ROBOT_JAIL = os.environ.get('ROBOT_JAIL', '') gp.dprint_vars(ROBOT_TEST_BASE_DIR_PATH, ROBOT_TEST_RUNNING_FROM_SB, ROBOT_JAIL, robot_jail) # Save PATH and PYTHONPATH to be restored later. os.environ["SAVED_PYTHONPATH"] = os.environ.get("PYTHONPATH", "") os.environ["SAVED_PATH"] = os.environ.get("PATH", "") if robot_jail: # Make sure required programs like python and robot can be found in the new restricted PATH. required_programs = "python robot" # It is expected that there will be a "python" program in the tool base bin path which is really a # link to select_version. Ditto for "robot". Call each with the --print_only option to get the # paths to the "real" programs. cmd_buf = "for program in " + required_programs \ + " ; do dirname $(${program} --print_only) ; done 2>/dev/null" rc, out_buf = gc.shell_cmd(cmd_buf, quiet=1, print_output=0) PYTHONPATH = ROBOT_TEST_BASE_DIR_PATH + "lib" NEW_PATH_LIST = [ROBOT_TEST_BASE_DIR_PATH + "bin"] NEW_PATH_LIST.extend(list(set(out_buf.rstrip("\n").split("\n")))) NEW_PATH_LIST.extend([ "/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin", OPENBMCTOOL_DIR_PATH.rstrip('/') ]) PATH = ":".join(NEW_PATH_LIST) else: PYTHONPATH = os.environ.get('PYTHONPATH', '') + ":" +\ ROBOT_TEST_BASE_DIR_PATH + "lib" PATH = os.environ.get('PATH', '') + ":" + ROBOT_TEST_BASE_DIR_PATH +\ "bin" + ":" + OPENBMCTOOL_DIR_PATH.rstrip('/') os.environ['PYTHONPATH'] = PYTHONPATH os.environ['PATH'] = PATH gp.dprint_vars(PATH, PYTHONPATH) os.environ['FFDC_DIR_PATH_STYLE'] = os.environ.get('FFDC_DIR_PATH_STYLE', '1') gp.qpissuing(robot_cmd_buf, test_mode) if test_mode: os.environ["PATH"] = os.environ.get("SAVED_PATH", "") os.environ["PYTHONPATH"] = os.environ.get("SAVED_PYTHONPATH", "") return True if quiet: DEVNULL = open(os.devnull, 'wb') stdout = DEVNULL else: stdout = None sub_proc = subprocess.Popen(robot_cmd_buf, stdout=stdout, shell=True) sub_proc.communicate() shell_rc = sub_proc.returncode os.environ["PATH"] = os.environ.get("SAVED_PATH", "") os.environ["PYTHONPATH"] = os.environ.get("SAVED_PYTHONPATH", "") gcr_last_robot_rc = shell_rc process_robot_output_files() if shell_rc != 0: gp.print_var(shell_rc, gp.hexa()) return False return True
def shell_cmd(command_string, quiet=None, print_output=None, show_err=1, test_mode=0, time_out=None, max_attempts=1, retry_sleep_time=5, valid_rcs=[0], ignore_err=None, return_stderr=0, fork=0, error_regexes=None): r""" Run the given command string in a shell and return a tuple consisting of the shell return code and the output. Description of argument(s): command_string The command string to be run in a shell (e.g. "ls /tmp"). quiet If set to 0, this function will print "Issuing: <cmd string>" to stdout. When the quiet argument is set to None, this function will assign a default value by searching upward in the stack for the quiet variable value. If no such value is found, quiet is set to 0. print_output If this is set, this function will print the stdout/stderr generated by the shell command to stdout. show_err If show_err is set, this function will print a standardized error report if the shell command fails (i.e. if the shell command returns a shell_rc that is not in valid_rcs). Note: Error text is only printed if ALL attempts to run the command_string fail. In other words, if the command execution is ultimately successful, initial failures are hidden. test_mode If test_mode is set, this function will not actually run the command. If print_output is also set, this function will print "(test_mode) Issuing: <cmd string>" to stdout. A caller should call shell_cmd directly if they wish to have the command string run unconditionally. They should call the t_shell_cmd wrapper (defined below) if they wish to run the command string only if the prevailing test_mode variable is set to 0. time_out A time-out value expressed in seconds. If the command string has not finished executing within <time_out> seconds, it will be halted and counted as an error. max_attempts The max number of attempts that should be made to run the command string. retry_sleep_time The number of seconds to sleep between attempts. valid_rcs A list of integers indicating which shell_rc values are not to be considered errors. ignore_err Ignore error means that a failure encountered by running the command string will not be raised as a python exception. When the ignore_err argument is set to None, this function will assign a default value by searching upward in the stack for the ignore_err variable value. If no such value is found, ignore_err is set to 1. return_stderr If return_stderr is set, this function will process the stdout and stderr streams from the shell command separately. In such a case, the tuple returned by this function will consist of three values rather than just two: rc, stdout, stderr. fork Run the command string asynchronously (i.e. don't wait for status of the child process and don't try to get stdout/stderr) and return the Popen object created by the subprocess.popen() function. See the kill_cmd function for details on how to process the popen object. error_regexes A list of regular expressions to be used to identify errors in the command output. If there is a match for any of these regular expressions, the command will be considered a failure and the shell_rc will be set to -1. For example, if error_regexes = ['ERROR:'] and the command output contains 'ERROR: Unrecognized option', it will be counted as an error even if the command returned 0. This is useful when running commands that do not always return non-zero on error. """ err_msg = gv.valid_value(command_string) if err_msg: raise ValueError(err_msg) # Assign default values to some of the arguments to this function. quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0))) print_output = int(gm.dft(print_output, not quiet)) show_err = int(show_err) ignore_err = int(gm.dft(ignore_err, gp.get_stack_var('ignore_err', 1))) gp.qprint_issuing(command_string, test_mode) if test_mode: return (0, "", "") if return_stderr else (0, "") # Convert a string python dictionary definition to a dictionary. valid_rcs = fa.source_to_object(valid_rcs) # Convert each list entry to a signed value. valid_rcs = [gm.to_signed(x) for x in valid_rcs] stderr = subprocess.PIPE if return_stderr else subprocess.STDOUT # Write all output to func_out_history_buf rather than directly to stdout. This allows us to decide # what to print after all attempts to run the command string have been made. func_out_history_buf will # contain the complete history from the current invocation of this function. global command_timed_out command_timed_out = False func_out_history_buf = "" for attempt_num in range(1, max_attempts + 1): sub_proc = subprocess.Popen(command_string, preexec_fn=os.setsid, bufsize=1, shell=True, universal_newlines=True, executable='/bin/bash', stdout=subprocess.PIPE, stderr=stderr) if fork: return sub_proc if time_out: command_timed_out = False # Designate a SIGALRM handling function and set alarm. signal.signal(signal.SIGALRM, shell_cmd_timed_out) signal.alarm(time_out) try: stdout_buf, stderr_buf = sub_proc.communicate() except IOError: command_timed_out = True # Restore the original SIGALRM handler and clear the alarm. signal.signal(signal.SIGALRM, original_sigalrm_handler) signal.alarm(0) # Output from this loop iteration is written to func_out_buf for later processing. This can include # stdout, stderr and our own error messages. func_out_buf = "" if print_output: if return_stderr: func_out_buf += stderr_buf func_out_buf += stdout_buf shell_rc = sub_proc.returncode if shell_rc in valid_rcs: # Check output for text indicating there is an error. if error_regexes and re.match('|'.join(error_regexes), stdout_buf): shell_rc = -1 else: break err_msg = "The prior shell command failed.\n" err_msg += gp.sprint_var(attempt_num) err_msg += gp.sprint_vars(command_string, command_timed_out, time_out) err_msg += gp.sprint_varx("child_pid", sub_proc.pid) err_msg += gp.sprint_vars(shell_rc, valid_rcs, fmt=gp.hexa()) if error_regexes: err_msg += gp.sprint_vars(error_regexes) if not print_output: if return_stderr: err_msg += "stderr_buf:\n" + stderr_buf err_msg += "stdout_buf:\n" + stdout_buf if show_err: func_out_buf += gp.sprint_error_report(err_msg) if attempt_num < max_attempts: cmd_buf = "time.sleep(" + str(retry_sleep_time) + ")" if show_err: func_out_buf += gp.sprint_issuing(cmd_buf) exec(cmd_buf) func_out_history_buf += func_out_buf if shell_rc in valid_rcs: gp.gp_print(func_out_buf) else: if show_err: gp.gp_print(func_out_history_buf, stream='stderr') else: # There is no error information to show so just print output from last loop iteration. gp.gp_print(func_out_buf) if not ignore_err: # If the caller has already asked to show error info, avoid repeating that in the failure message. err_msg = "The prior shell command failed.\n" if show_err \ else err_msg if robot_env: BuiltIn().fail(err_msg) else: raise ValueError(err_msg) return (shell_rc, stdout_buf, stderr_buf) if return_stderr \ else (shell_rc, stdout_buf)