def runOc(*args, **kwargs): return sh.oc(*args, **kwargs, _tee=True)
def oc(*args, **kwargs): """ Run 'sh.oc' and print the command, show output, catch errors, etc. Optional kwargs: _reraise: if ErrorReturnCode is hit, don't exit, re-raise it _exit_on_err: sys.exit(1) if this command fails (default True) _silent: don't print command or resulting stdout (default False) _ignore_immutable: ignore errors related to immutable objects (default True) Returns: None if cmd fails and _exit_on_err is False command output (str) if command succeeds """ _exit_on_err = kwargs.pop("_exit_on_err", True) _silent = kwargs.pop("_silent", False) _reraise = kwargs.pop("_reraise", False) _ignore_immutable = kwargs.pop("_ignore_immutable", True) kwargs["_bg_exc"] = True # Format the cmd args/kwargs for log printing before the command is run # Maybe 'sh' provides an easy way to do this...? cmd_args = " ".join([str(arg) for arg in args if arg is not None]) cmd_kwargs = [] for key, val in kwargs.items(): if key.startswith("_"): continue if len(key) > 1: cmd_kwargs.append("--{} {}".format(key, val)) else: cmd_kwargs.append("-{} {}".format(key, val)) cmd_kwargs = " ".join(cmd_kwargs) if not _silent: log.info("Running command: oc %s %s", cmd_args, cmd_kwargs) err_lines = [] out_lines = [] def _err_line_handler(line): log.info(" |stderr| %s", line.rstrip()) err_lines.append(line) def _out_line_handler(line): if not _silent: log.info(" |stdout| %s", line.rstrip()) out_lines.append(line) try: return sh.oc(*args, **kwargs, _tee=True, _out=_out_line_handler, _err=_err_line_handler).wait() except ErrorReturnCode: immutable_errors_only = False # Ignore warnings that are printed to stderr err_lines = [ line for line in err_lines if not line.lstrip().startswith("Warning:") ] if err_lines: immutable_errors_only = all( "field is immutable after creation" in line for line in err_lines) if immutable_errors_only and _ignore_immutable: log.warning("Ignoring immutable field errors") elif _reraise: raise elif _exit_on_err: log.error("Command failed! Aborting.") sys.exit(1) else: log.warning("Non-zero return code ignored")
def _exec_oc(*args, **kwargs): _silent = kwargs.pop("_silent", False) _ignore_immutable = kwargs.pop("_ignore_immutable", True) _retry_conflicts = kwargs.pop("_retry_conflicts", True) _stdout_log_prefix = kwargs.pop("_stdout_log_prefix", " |stdout| ") _stderr_log_prefix = kwargs.pop("_stderr_log_prefix", " |stderr| ") kwargs["_bg"] = True kwargs["_bg_exc"] = False err_lines = [] out_lines = [] def _err_line_handler(line, _, process): threading.current_thread().name = f"pid-{process.pid}" if not _silent: log.info("%s%s", _stderr_log_prefix, line.rstrip()) err_lines.append(line) def _out_line_handler(line, _, process): threading.current_thread().name = f"pid-{process.pid}" if not _silent: log.info("%s%s", _stdout_log_prefix, line.rstrip()) out_lines.append(line) retries = 3 last_err = None for count in range(1, retries + 1): cmd = sh.oc(*args, **kwargs, _tee=True, _out=_out_line_handler, _err=_err_line_handler) if not _silent: cmd_args, cmd_kwargs = _get_logging_args(args, kwargs) log.info("running (pid %d): oc %s %s", cmd.pid, cmd_args, cmd_kwargs) try: return cmd.wait() except ErrorReturnCode as err: # Sometimes stdout/stderr is empty in the exception even though we appended # data in the callback. Perhaps buffers are not being flushed ... so just # set the out lines/err lines we captured on the Exception before re-raising it by # re-init'ing the err and causing it to rebuild its message template. # # see https://github.com/amoffat/sh/blob/master/sh.py#L381 err.__init__( full_cmd=err.full_cmd, stdout="\n".join(out_lines).encode(), stderr="\n".join(err_lines).encode(), truncate=err.truncate, ) # Make these plain strings for easier exception handling err.stdout = "\n".join(out_lines) err.stderr = "\n".join(err_lines) last_err = err # Ignore warnings that are printed to stderr in our error analysis err_lines = [ line for line in err_lines if not line.lstrip().startswith("Warning:") ] # Check if these are errors we should handle if _ignore_immutable and _only_immutable_errors(err_lines): log.warning("Ignoring immutable field errors") break elif _retry_conflicts and _conflicts_found(err_lines): log.warning( "Hit resource conflict, retrying in 1 sec (attempt %d/%d)", count, retries, ) time.sleep(1) continue # Bail if not raise last_err else: log.error("Retried %d times, giving up", retries) raise last_err
def _exec_oc(*args, **kwargs): _silent = kwargs.pop("_silent", False) _ignore_immutable = kwargs.pop("_ignore_immutable", True) _retry_conflicts = kwargs.pop("_retry_conflicts", True) kwargs["_bg_exc"] = True err_lines = [] out_lines = [] def _err_line_handler(line): log.info(" |stderr| %s", line.rstrip()) err_lines.append(line) def _out_line_handler(line): if not _silent: log.info(" |stdout| %s", line.rstrip()) out_lines.append(line) retries = 3 last_err = None for count in range(1, retries + 1): try: if not _silent: cmd_args, cmd_kwargs = _get_logging_args(args, kwargs) log.info("Running command: oc %s %s", cmd_args, cmd_kwargs) return sh.oc(*args, **kwargs, _tee=True, _out=_out_line_handler, _err=_err_line_handler).wait() except ErrorReturnCode as err: # Sometimes stdout/stderr is empty in the exception even though we appended # data in the callback. Perhaps buffers are not being flushed ... so just # set the out lines/err lines we captured on the Exception before re-raising it err.stdout = "\n".join(out_lines) err.stderr = "\n".join(err_lines) last_err = err # Ignore warnings that are printed to stderr in our error analysis err_lines = [ line for line in err_lines if not line.lstrip().startswith("Warning:") ] # Check if these are errors we should handle if _ignore_immutable and _only_immutable_errors(err_lines): log.warning("Ignoring immutable field errors") break elif _retry_conflicts and _conflicts_found(err_lines): log.warning( "Hit resource conflict, retrying in 1 sec (attempt %d/%d)", count, retries) time.sleep(1) continue # Bail if not raise else: log.error("Retried %d times, giving up", retries) raise last_err