def delete(path, fatal=True, logger=UNSET, dryrun=UNSET): """ Args: path (str | Path | None): Path to file or folder to delete fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter dryrun (bool | UNSET | None): Optionally override current dryrun setting Returns: (int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed """ path = resolved_path(path) islink = path and os.path.islink(path) if not islink and (not path or not os.path.exists(path)): return 0 if _R.hdry(dryrun, logger, "delete %s" % short(path)): return 1 try: _do_delete(path, islink, fatal) _R.hlog(logger, "Deleted %s" % short(path)) return 1 except Exception as e: return abort("Can't delete %s" % short(path), exc_info=e, return_value=-1, fatal=fatal, logger=logger)
def make_executable(path, fatal=True, logger=UNSET, dryrun=UNSET): """ Args: path (str): chmod file with 'path' as executable fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter dryrun (bool | UNSET | None): Optionally override current dryrun setting Returns: (int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed """ if is_executable(path): return 0 if _R.hdry(dryrun, logger, "make %s executable" % short(path)): return 1 if not os.path.exists(path): return abort("%s does not exist, can't make it executable" % short(path), return_value=-1, fatal=fatal, logger=logger) try: os.chmod(path, 0o755) # nosec _R.hlog(logger, "Made '%s' executable" % short(path)) return 1 except Exception as e: return abort("Can't chmod %s" % short(path), exc_info=e, return_value=-1, fatal=fatal, logger=logger)
def filesize(*paths, logger=False): """ Args: *paths (str | Path | None): Paths to files/folders logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter Returns: (int): File size in bytes """ size = 0 for path in flattened(paths, unique=True): path = to_path(path) if path and path.exists() and not path.is_symlink(): if path.is_dir(): for sf in path.iterdir(): size += filesize(sf) elif path.is_file(): try: size += path.stat().st_size except Exception as e: # pragma: no cover, ignore cases like permission denied, file name too long, etc _R.hlog(logger, "Can't stat %s: %s" % (short(path), short(e, size=32))) return size
def match(self, expected, stdout=None, stderr=None, regex=None): """ Args: expected (str | re.Pattern): Message to find in self.logged stdout (bool | None): Look at stdout (default: yes, if captured) stderr (bool | None): Look at stderr (default: yes, if captured) regex (int | bool | None): Specify whether 'expected' should be a regex Returns: (Match | None): Match found, if any """ if stdout is None and stderr is None: # By default, look at stdout/stderr only stdout = stderr = True assert expected, "No 'expected' provided" assert self.exit_code is not None, "run() was not called yet" captures = [stdout and self.logged.stdout, stderr and self.logged.stderr] captures = [c for c in captures if c is not None and c is not False] assert captures, "No captures specified" if not any(c for c in captures): # There was no output at all return None if not isinstance(regex, bool) and isinstance(regex, int): flags = regex regex = True else: flags = 0 if isinstance(expected, str) and "..." in expected and not isinstance(regex, bool): regex = True expected = expected.replace("...", ".+") if not isinstance(expected, str): # Assume regex, no easy way to verify isinstance(expected, re.Pattern) for python < 3.7 regex = expected elif regex: regex = re.compile("(.{0,32})(%s)(.{0,32})" % expected, flags=flags) for c in captures: contents = c.contents() if regex: m = regex.search(contents) if m: if m.groups(): return Match(c, m.group(2), pre=m.group(1), post=m.group(3)) return Match(c, m.group(0)) elif expected in contents: i = contents.index(expected) pre = short(contents[:i], size=32) post = short(contents[i + len(expected):], size=32) return Match(c, expected, pre=pre, post=post)
def readlines(path, first=None, errors="ignore", fatal=False, logger=False): """ Args: path (str | Path | None): Path to file to read lines from first (int | None): Return only the 'first' lines when specified errors (str | None): Optional string specifying how encoding errors are to be handled fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter Yields: (str): Lines read, newlines and trailing spaces stripped """ try: with io.open(resolved_path(path), errors=errors) as fh: if not first: first = -1 for line in fh: if first == 0: return yield decode(line).rstrip() first -= 1 except Exception as e: message = "Can't read %s" % short(path) if fatal: abort(_R.actual_message(message), exc_info=e, fatal=fatal, logger=logger) _R.hlog(logger, message, exc_info=e)
def download(self, url, destination, fatal=True, logger=UNSET, dryrun=UNSET, **kwargs): """ Args: url (str): URL of resource to download (may be absolute, or relative to self.base_url) Use #sha256=... or #sha512=... at the end of the url to ensure content is validated against given checksum destination (str | Path): Path to local file where to store the download fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter dryrun (bool | UNSET | None): Optionally override current dryrun setting **kwargs: Passed through to underlying client Returns: (RestResponse): Response from underlying call """ hash_algo, hash_checksum, url = self._decomposed_checksum_url(url) response = self._get_response("GET", url, fatal, logger, dryrun=dryrun, action="download", **kwargs) if response.ok and not _R.resolved_dryrun(dryrun): destination = to_path(destination) ensure_folder(destination.parent, fatal=fatal, logger=None) with open(destination, "wb") as fh: fh.write(response.content) if hash_checksum: downloaded_checksum = checksum(destination, hash=hash_algo) if downloaded_checksum != hash_checksum: delete(destination, fatal=False, logger=logger) msg = "%s differs for %s: expecting %s, got %s" % (hash_algo, short(destination), hash_checksum, downloaded_checksum) return abort(msg, fatal=fatal, logger=logger) return response
def _single_diag(sources, border, align, style, missing, columns): table = PrettyTable(2, border=border) table.header[0].align = align table.header[1].style = style col1 = 0 rows = [] for source in sources: if callable(source): source = source() if isinstance(source, dict): source = sorted(source.items()) elif not is_iterable(source): source = [source] for row in source: if not isinstance(row, (tuple, list)): row = (row, "") row = [_represented_cell(s, missing) for s in row] col1 = max(col1, len(row[0])) rows.append(row) columns = max(columns - col1 - 5, 10) for row in rows: if len(row) == 2 and row[1]: row[1] = short(row[1], size=columns, uncolor=True) table.add_row(row) return table.get_string()
def ensure_folder(path, clean=False, fatal=True, logger=UNSET, dryrun=UNSET): """Ensure folder with 'path' exists Args: path (str | Path | None): Path to file or folder clean (bool): True: If True, ensure folder is clean (delete any file/folder it may have) fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter dryrun (bool | UNSET | None): Optionally override current dryrun setting Returns: (int): In non-fatal mode, >=1: successfully done, 0: was no-op, -1: failed """ path = resolved_path(path) if not path: return 0 if os.path.isdir(path): if not clean: return 0 cleaned = 0 for fname in os.listdir(path): cleaned += delete(os.path.join(path, fname), fatal=fatal, logger=None, dryrun=dryrun) if cleaned: msg = "%s from %s" % (_R.lc.rm.plural(cleaned, "file"), short(path)) if not _R.hdry(dryrun, logger, "clean %s" % msg): _R.hlog(logger, "Cleaned %s" % msg) return cleaned if _R.hdry(dryrun, logger, "create %s" % short(path)): return 1 try: os.makedirs(path) _R.hlog(logger, "Created folder %s" % short(path)) return 1 except Exception as e: return abort("Can't create folder %s" % short(path), exc_info=e, return_value=-1, fatal=fatal, logger=logger)
def scan_exe(cls, exe): """ Args: exe (str): Path to python executable Returns: (PyInstallInfo): Extracted info """ r = run(exe, cls.get_pv(), dryrun=False, fatal=False, logger=None) if not r.succeeded: return PyInstallInfo(problem=short(r.full_output)) try: lines = r.output.strip().splitlines() if len(lines) != 3: return PyInstallInfo( problem="introspection yielded %s lines instead of 3" % len(lines)) version, sys_prefix, base_prefix = lines return PyInstallInfo(version, sys_prefix, base_prefix) except Exception as e: # pragma: no cover return PyInstallInfo(problem="introspection error: %s" % short(e))
def add_text(self, line, columns): """(int): size of text added to 'line' (lock already acquired)""" text = self.current_text if not text or columns <= 0: return 0 if self.adapter is not None: text = self.adapter(text) size = len(text) if self.adapter is not None or size > columns: text = short(text, size=columns) text = _R.colored(text, self.color) line.append(text) return size
def __call__(self, text, size=None): """ Allows for convenient call of the form: >>> import runez >>> runez.blue("foo") 'foo' """ if size: text = short(text, size=size) else: text = stringified(text) if not text: return "" return self.rendered(text)
def shortened_program(program, args): if program: base = os.path.basename(program) if args and base in ("python", "python3"): if args[0] == "-m": return args[1], args[2:] if args[0].startswith("-m"): return args[0][2:], args[1:] if args[0].endswith("__main__.py"): program = args[0] program = os.path.dirname(os.path.abspath(program)) return os.path.basename(program), args[1:] elif SYS_INFO.venv_bin_folder and program.startswith( SYS_INFO.venv_bin_folder): return base, args return short(program), args
def write(path, contents, fatal=True, logger=UNSET, dryrun=UNSET): """Write `contents` to file with `path` Args: path (str | Path | None): Path to file contents (str | bytes | None): Contents to write (only touch file if None) fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter dryrun (bool | UNSET | None): Optionally override current dryrun setting Returns: (int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed """ if not path: return 0 path = resolved_path(path) byte_size = _R.lc.rm.represented_bytesize(len(contents), unit="bytes") if contents else "" def dryrun_msg(): return "%s %s" % ("write %s to" % byte_size if byte_size else "touch", short(path)) if _R.hdry(dryrun, logger, dryrun_msg): return 1 ensure_folder(parent_folder(path), fatal=fatal, logger=None, dryrun=dryrun) try: mode = "wb" if isinstance(contents, bytes) else "wt" with io.open(path, mode) as fh: if contents is None: os.utime(path, None) else: fh.write(contents) _R.hlog(logger, "%s %s" % ("Wrote %s to" % byte_size if byte_size else "Touched", short(path))) return 1 except Exception as e: return abort("Can't write to %s" % short(path), exc_info=e, return_value=-1, fatal=fatal, logger=logger)
def run_description(self, short_exe=UNSET): """ Args: short_exe (str | bool | None): Try to log a compact representation of executable Returns: (str): Description for self.program run with self.args """ program = self.program args = self.args if short_exe is UNSET and program and SYS_INFO.venv_bin_folder: short_exe = program.startswith(SYS_INFO.venv_bin_folder) if isinstance(short_exe, str): program = short_exe elif short_exe is True: program, args = self.shortened_program(program, args) else: program = short(program) return "%s %s" % (program, quoted(args)) if args and program else program
def __repr__(self): return short(self.canonical)
def __repr__(self): text = joined(self.problem or self.spec, self.is_invoker and "invoker", delimiter=", ") return "%s [%s]" % (short(self.short_name), text)
def run_cmds(cls, prog=None): """To be called from one's main() Args: prog (str | None): The name of the program (default: sys.argv[0]) """ from runez.render import PrettyTable caller = find_caller() package = caller.package_name # Will fail if no caller could be found (intentional) available_commands = {} for name, func in caller.globals(prefix="cmd_"): name = name[4:].replace("_", "-") available_commands[name] = func if not prog: if package: prog = "python -m%s" % package if caller.is_main else package elif caller.basename in ("__init__.py", "__main__.py"): prog = short(caller.folder) epilog = PrettyTable(2) epilog.header[0].style = "bold" for cmd, func in available_commands.items(): epilog.add_row(" " + cmd, first_line(func.__doc__, default="")) epilog = "Available commands:\n%s" % epilog cls._prog = prog or package parser = cls.parser(epilog=epilog, help=caller.module_docstring, prog=prog) if cls.version and package: parser.add_argument(*cls.version, action="version", version=get_version(package), help="Show version and exit") if cls.color: parser.add_argument(*cls.color, action="store_true", help="Do not use colors (even if on tty)") if cls.debug: parser.add_argument(*cls.debug, action="store_true", help="Show debugging information") if cls.dryrun: parser.add_argument(*cls.dryrun, action="store_true", help="Perform a dryrun") parser.add_argument("command", choices=available_commands, metavar="command", help="Command to run") parser.add_argument("args", nargs=argparse.REMAINDER, help="Passed-through to command") args = parser.parse_args() if cls.console_format or hasattr(args, "debug") or hasattr( args, "dryrun"): LogManager.setup( debug=getattr(args, "debug", UNSET), dryrun=getattr(args, "dryrun", UNSET), console_format=cls.console_format, console_level=cls.console_level, default_logger=cls.default_logger, locations=cls.log_locations, ) color = getattr(args, "no_color", None) if color is not None: ColorManager.activate_colors(enable=not color) try: func = available_commands[args.command] with TempArgv(args.args): func() except KeyboardInterrupt: # pragma: no cover sys.stderr.write("\nAborted\n") sys.exit(1)
def __repr__(self): return "%s [%s]" % (self.scanner_name, short(self.location))
def _file_op(source, destination, func, overwrite, fatal, logger, dryrun, must_exist=True, ignore=None, **extra): """Call func(source, destination) Args: source (str | None): Source file or folder destination (str | None): Destination file or folder func (callable): Implementation function overwrite (bool | None): True: replace existing, False: fail if destination exists, None: no destination check fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter dryrun (bool | UNSET | None): Optionally override current dryrun setting must_exist (bool): If True, verify that source does indeed exist ignore (callable | list | str | None): Names to be ignored **extra: Passed-through to 'func' Returns: (int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed """ if not source or not destination or source == destination: return 0 action = func.__name__[1:] indicator = "<-" if action == "symlink" else "->" description = "%s %s %s %s" % (action, short(source), indicator, short(destination)) psource = parent_folder(source) pdest = resolved_path(destination) if psource.startswith(pdest): message = "Can't %s: source contained in destination" % description return abort(message, return_value=-1, fatal=fatal, logger=logger) if _R.hdry(dryrun, logger, description): return 1 if must_exist and not (os.path.exists(source) or os.path.islink(source)): message = "%s does not exist, can't %s to %s" % (short(source), action.lower(), short(destination)) return abort(message, return_value=-1, fatal=fatal, logger=logger) if overwrite is not None: islink = os.path.islink(pdest) if islink or os.path.exists(pdest): if not overwrite: message = "%s exists, can't %s" % (short(destination), action.lower()) return abort(message, return_value=-1, fatal=fatal, logger=logger) _do_delete(pdest, islink, fatal) try: # Ensure parent folder exists ensure_folder(parent_folder(destination), fatal=fatal, logger=None, dryrun=dryrun) _R.hlog(logger, "%s%s" % (description[0].upper(), description[1:])) if ignore is not None: if not callable(ignore): given = ignore def ignore(*_): return given extra["ignore"] = ignore func(source, destination, **extra) return 1 except Exception as e: return abort("Can't %s" % description, exc_info=e, return_value=-1, fatal=fatal, logger=logger)
def dryrun_msg(): return "%s %s" % ("write %s to" % byte_size if byte_size else "touch", short(path))
def run(program, *args, background=False, fatal=True, logger=UNSET, dryrun=UNSET, short_exe=UNSET, passthrough=False, path_env=None, strip="\r\n", stdout=subprocess.PIPE, stderr=subprocess.PIPE, **popen_args): """Run 'program' with 'args' Args: program (str | pathlib.Path): Program to run (full path, or basename) *args: Command line args to call 'program' with background (bool): When True, background the spawned process (detach from console and current process) fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter dryrun (bool | UNSET | None): Optionally override current dryrun setting short_exe (str | bool | None): Try to log a compact representation of executable passthrough (bool | file | None): If True-ish, pass-through stderr/stdout in addition to capturing it as well as 'passthrough' itself if it has a write() function path_env (dict | None): Allows to inject PATH-like env vars, see `_added_env_paths()` strip (str | bool | None): If provided, `strip()` the captured output [default: strip "\n" newlines] stdout (int | IO[Any] | None): Passed-through to subprocess.Popen, [default: subprocess.PIPE] stderr (int | IO[Any] | None): Passed-through to subprocess.Popen, [default: subprocess.PIPE] **popen_args: Passed through to `subprocess.Popen` Returns: (RunResult): Run outcome, use .failed, .succeeded, .output, .error etc to inspect the outcome """ if path_env: popen_args["env"] = _added_env_paths(path_env, env=popen_args.get("env")) args = flattened(args, shellify=True) full_path = which(program) result = RunResult(audit=RunAudit(full_path or program, args, popen_args)) description = result.audit.run_description(short_exe=short_exe) if background: description += " &" abort_logger = None if logger is None else UNSET if logger is True or logger is print: # When logger is True, we just print() the message, so we may as well color it nicely description = _R.colored(description, "bold") if _R.hdry(dryrun, logger, "run: %s" % description): result.audit.dryrun = True result.exit_code = 0 if stdout is not None: result.output = "[dryrun] %s" % description # Properly simulate a successful run if stdout is not None: result.error = "" return result if not full_path: if program and os.path.basename(program) == program: result.error = "%s is not installed (PATH=%s)" % ( short(program), short(os.environ.get("PATH"))) else: result.error = "%s is not an executable" % short(program) return abort(result.error, return_value=result, fatal=fatal, logger=abort_logger) _R.hlog(logger, "Running: %s" % description) if background: child_pid = daemonize() if child_pid: result.pid = child_pid # In parent process, we just report a successful run (we don't wait/check on background process) result.exit_code = 0 return result fatal = False # pragma: no cover, non-fatal mode in background process (there is no more console etc to report anything) with _WrappedArgs([full_path] + args) as wrapped_args: try: p, out, err = _run_popen(wrapped_args, popen_args, passthrough, fatal, stdout, stderr) result.output = decode(out, strip=strip) result.error = decode(err, strip=strip) result.pid = p.pid result.exit_code = p.returncode except Exception as e: if fatal: # Don't re-wrap with an abort(), let original stacktrace show through raise result.exc_info = e if not result.error: result.error = "%s failed: %s" % ( short(program), repr(e) if isinstance(e, OSError) else e) if fatal and result.exit_code: base_message = "%s exited with code %s" % (short(program), result.exit_code) if passthrough and (result.output or result.error): exception = _R.abort_exception(override=fatal) if exception is SystemExit: raise SystemExit(result.exit_code) if isinstance(exception, type) and issubclass( exception, BaseException): raise exception(base_message) message = [] if abort_logger is not None and not passthrough: # Log full output, unless user explicitly turned it off message.append("Run failed: %s" % description) if result.error: message.append("\nstderr:\n%s" % result.error) if result.output: message.append("\nstdout:\n%s" % result.output) message.append(base_message) abort("\n".join(message), code=result.exit_code, exc_info=result.exc_info, fatal=fatal, logger=abort_logger) if background: os._exit( result.exit_code ) # pragma: no cover, simply exit forked process (don't go back to caller) return result
def to_path(path, no_spaces=False): """ Args: path (str | Path): Path to convert no_spaces (type | bool | None): If True-ish, abort if 'path' contains a space Returns: (Path | None): Converted to `Path` object, if necessary """ if no_spaces and " " in str(path): abort("Refusing path with space (not worth escaping all the things to make this work): '%s'" % short(path), fatal=no_spaces) if isinstance(path, Path): return path if path is not None: if path: path = os.path.expanduser(path) return Path(path)
def __str__(self): text = _R.colored(self.problem, "red") or self.spec text = joined(text, self.is_invoker and _R.colored("invoker", "green"), delimiter=", ") return "%s [%s]" % (_R.colored(short(self.short_name), "bold"), text)