def delete(path, fatal=True, logger=UNSET, dryrun=UNSET):
"""
Args:
path (str | Path | None): Path to file or folder to delete
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
Returns:
(int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed
"""
path = resolved_path(path)
islink = path and os.path.islink(path)
if not islink and (not path or not os.path.exists(path)):
return 0
if _R.hdry(dryrun, logger, "delete %s" % short(path)):
return 1
try:
_do_delete(path, islink, fatal)
_R.hlog(logger, "Deleted %s" % short(path))
return 1
except Exception as e:
return abort("Can't delete %s" % short(path), exc_info=e, return_value=-1, fatal=fatal, logger=logger)
def make_executable(path, fatal=True, logger=UNSET, dryrun=UNSET):
"""
Args:
path (str): chmod file with 'path' as executable
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
Returns:
(int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed
"""
if is_executable(path):
return 0
if _R.hdry(dryrun, logger, "make %s executable" % short(path)):
return 1
if not os.path.exists(path):
return abort("%s does not exist, can't make it executable" %
short(path),
return_value=-1,
fatal=fatal,
logger=logger)
try:
os.chmod(path, 0o755) # nosec
_R.hlog(logger, "Made '%s' executable" % short(path))
return 1
except Exception as e:
return abort("Can't chmod %s" % short(path),
exc_info=e,
return_value=-1,
fatal=fatal,
logger=logger)
def filesize(*paths, logger=False):
"""
Args:
*paths (str | Path | None): Paths to files/folders
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
Returns:
(int): File size in bytes
"""
size = 0
for path in flattened(paths, unique=True):
path = to_path(path)
if path and path.exists() and not path.is_symlink():
if path.is_dir():
for sf in path.iterdir():
size += filesize(sf)
elif path.is_file():
try:
size += path.stat().st_size
except Exception as e: # pragma: no cover, ignore cases like permission denied, file name too long, etc
_R.hlog(logger, "Can't stat %s: %s" % (short(path), short(e, size=32)))
return size
def match(self, expected, stdout=None, stderr=None, regex=None):
"""
Args:
expected (str | re.Pattern): Message to find in self.logged
stdout (bool | None): Look at stdout (default: yes, if captured)
stderr (bool | None): Look at stderr (default: yes, if captured)
regex (int | bool | None): Specify whether 'expected' should be a regex
Returns:
(Match | None): Match found, if any
"""
if stdout is None and stderr is None:
# By default, look at stdout/stderr only
stdout = stderr = True
assert expected, "No 'expected' provided"
assert self.exit_code is not None, "run() was not called yet"
captures = [stdout and self.logged.stdout, stderr and self.logged.stderr]
captures = [c for c in captures if c is not None and c is not False]
assert captures, "No captures specified"
if not any(c for c in captures):
# There was no output at all
return None
if not isinstance(regex, bool) and isinstance(regex, int):
flags = regex
regex = True
else:
flags = 0
if isinstance(expected, str) and "..." in expected and not isinstance(regex, bool):
regex = True
expected = expected.replace("...", ".+")
if not isinstance(expected, str):
# Assume regex, no easy way to verify isinstance(expected, re.Pattern) for python < 3.7
regex = expected
elif regex:
regex = re.compile("(.{0,32})(%s)(.{0,32})" % expected, flags=flags)
for c in captures:
contents = c.contents()
if regex:
m = regex.search(contents)
if m:
if m.groups():
return Match(c, m.group(2), pre=m.group(1), post=m.group(3))
return Match(c, m.group(0))
elif expected in contents:
i = contents.index(expected)
pre = short(contents[:i], size=32)
post = short(contents[i + len(expected):], size=32)
return Match(c, expected, pre=pre, post=post)
def readlines(path, first=None, errors="ignore", fatal=False, logger=False):
"""
Args:
path (str | Path | None): Path to file to read lines from
first (int | None): Return only the 'first' lines when specified
errors (str | None): Optional string specifying how encoding errors are to be handled
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
Yields:
(str): Lines read, newlines and trailing spaces stripped
"""
try:
with io.open(resolved_path(path), errors=errors) as fh:
if not first:
first = -1
for line in fh:
if first == 0:
return
yield decode(line).rstrip()
first -= 1
except Exception as e:
message = "Can't read %s" % short(path)
if fatal:
abort(_R.actual_message(message), exc_info=e, fatal=fatal, logger=logger)
_R.hlog(logger, message, exc_info=e)
def download(self, url, destination, fatal=True, logger=UNSET, dryrun=UNSET, **kwargs):
"""
Args:
url (str): URL of resource to download (may be absolute, or relative to self.base_url)
Use #sha256=... or #sha512=... at the end of the url to ensure content is validated against given checksum
destination (str | Path): Path to local file where to store the download
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
**kwargs: Passed through to underlying client
Returns:
(RestResponse): Response from underlying call
"""
hash_algo, hash_checksum, url = self._decomposed_checksum_url(url)
response = self._get_response("GET", url, fatal, logger, dryrun=dryrun, action="download", **kwargs)
if response.ok and not _R.resolved_dryrun(dryrun):
destination = to_path(destination)
ensure_folder(destination.parent, fatal=fatal, logger=None)
with open(destination, "wb") as fh:
fh.write(response.content)
if hash_checksum:
downloaded_checksum = checksum(destination, hash=hash_algo)
if downloaded_checksum != hash_checksum:
delete(destination, fatal=False, logger=logger)
msg = "%s differs for %s: expecting %s, got %s" % (hash_algo, short(destination), hash_checksum, downloaded_checksum)
return abort(msg, fatal=fatal, logger=logger)
return response
def _single_diag(sources, border, align, style, missing, columns):
table = PrettyTable(2, border=border)
table.header[0].align = align
table.header[1].style = style
col1 = 0
rows = []
for source in sources:
if callable(source):
source = source()
if isinstance(source, dict):
source = sorted(source.items())
elif not is_iterable(source):
source = [source]
for row in source:
if not isinstance(row, (tuple, list)):
row = (row, "")
row = [_represented_cell(s, missing) for s in row]
col1 = max(col1, len(row[0]))
rows.append(row)
columns = max(columns - col1 - 5, 10)
for row in rows:
if len(row) == 2 and row[1]:
row[1] = short(row[1], size=columns, uncolor=True)
table.add_row(row)
return table.get_string()
def ensure_folder(path, clean=False, fatal=True, logger=UNSET, dryrun=UNSET):
"""Ensure folder with 'path' exists
Args:
path (str | Path | None): Path to file or folder
clean (bool): True: If True, ensure folder is clean (delete any file/folder it may have)
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
Returns:
(int): In non-fatal mode, >=1: successfully done, 0: was no-op, -1: failed
"""
path = resolved_path(path)
if not path:
return 0
if os.path.isdir(path):
if not clean:
return 0
cleaned = 0
for fname in os.listdir(path):
cleaned += delete(os.path.join(path, fname), fatal=fatal, logger=None, dryrun=dryrun)
if cleaned:
msg = "%s from %s" % (_R.lc.rm.plural(cleaned, "file"), short(path))
if not _R.hdry(dryrun, logger, "clean %s" % msg):
_R.hlog(logger, "Cleaned %s" % msg)
return cleaned
if _R.hdry(dryrun, logger, "create %s" % short(path)):
return 1
try:
os.makedirs(path)
_R.hlog(logger, "Created folder %s" % short(path))
return 1
except Exception as e:
return abort("Can't create folder %s" % short(path), exc_info=e, return_value=-1, fatal=fatal, logger=logger)
def scan_exe(cls, exe):
"""
Args:
exe (str): Path to python executable
Returns:
(PyInstallInfo): Extracted info
"""
r = run(exe, cls.get_pv(), dryrun=False, fatal=False, logger=None)
if not r.succeeded:
return PyInstallInfo(problem=short(r.full_output))
try:
lines = r.output.strip().splitlines()
if len(lines) != 3:
return PyInstallInfo(
problem="introspection yielded %s lines instead of 3" %
len(lines))
version, sys_prefix, base_prefix = lines
return PyInstallInfo(version, sys_prefix, base_prefix)
except Exception as e: # pragma: no cover
return PyInstallInfo(problem="introspection error: %s" % short(e))
def add_text(self, line, columns):
"""(int): size of text added to 'line' (lock already acquired)"""
text = self.current_text
if not text or columns <= 0:
return 0
if self.adapter is not None:
text = self.adapter(text)
size = len(text)
if self.adapter is not None or size > columns:
text = short(text, size=columns)
text = _R.colored(text, self.color)
line.append(text)
return size
def __call__(self, text, size=None):
"""
Allows for convenient call of the form:
>>> import runez
>>> runez.blue("foo")
'foo'
"""
if size:
text = short(text, size=size)
else:
text = stringified(text)
if not text:
return ""
return self.rendered(text)
def shortened_program(program, args):
if program:
base = os.path.basename(program)
if args and base in ("python", "python3"):
if args[0] == "-m":
return args[1], args[2:]
if args[0].startswith("-m"):
return args[0][2:], args[1:]
if args[0].endswith("__main__.py"):
program = args[0]
program = os.path.dirname(os.path.abspath(program))
return os.path.basename(program), args[1:]
elif SYS_INFO.venv_bin_folder and program.startswith(
SYS_INFO.venv_bin_folder):
return base, args
return short(program), args
def write(path, contents, fatal=True, logger=UNSET, dryrun=UNSET):
"""Write `contents` to file with `path`
Args:
path (str | Path | None): Path to file
contents (str | bytes | None): Contents to write (only touch file if None)
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
Returns:
(int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed
"""
if not path:
return 0
path = resolved_path(path)
byte_size = _R.lc.rm.represented_bytesize(len(contents), unit="bytes") if contents else ""
def dryrun_msg():
return "%s %s" % ("write %s to" % byte_size if byte_size else "touch", short(path))
if _R.hdry(dryrun, logger, dryrun_msg):
return 1
ensure_folder(parent_folder(path), fatal=fatal, logger=None, dryrun=dryrun)
try:
mode = "wb" if isinstance(contents, bytes) else "wt"
with io.open(path, mode) as fh:
if contents is None:
os.utime(path, None)
else:
fh.write(contents)
_R.hlog(logger, "%s %s" % ("Wrote %s to" % byte_size if byte_size else "Touched", short(path)))
return 1
except Exception as e:
return abort("Can't write to %s" % short(path), exc_info=e, return_value=-1, fatal=fatal, logger=logger)
def run_description(self, short_exe=UNSET):
"""
Args:
short_exe (str | bool | None): Try to log a compact representation of executable
Returns:
(str): Description for self.program run with self.args
"""
program = self.program
args = self.args
if short_exe is UNSET and program and SYS_INFO.venv_bin_folder:
short_exe = program.startswith(SYS_INFO.venv_bin_folder)
if isinstance(short_exe, str):
program = short_exe
elif short_exe is True:
program, args = self.shortened_program(program, args)
else:
program = short(program)
return "%s %s" % (program,
quoted(args)) if args and program else program
def __repr__(self):
return short(self.canonical)
def __repr__(self):
text = joined(self.problem or self.spec,
self.is_invoker and "invoker",
delimiter=", ")
return "%s [%s]" % (short(self.short_name), text)
def run_cmds(cls, prog=None):
"""To be called from one's main()
Args:
prog (str | None): The name of the program (default: sys.argv[0])
"""
from runez.render import PrettyTable
caller = find_caller()
package = caller.package_name # Will fail if no caller could be found (intentional)
available_commands = {}
for name, func in caller.globals(prefix="cmd_"):
name = name[4:].replace("_", "-")
available_commands[name] = func
if not prog:
if package:
prog = "python -m%s" % package if caller.is_main else package
elif caller.basename in ("__init__.py", "__main__.py"):
prog = short(caller.folder)
epilog = PrettyTable(2)
epilog.header[0].style = "bold"
for cmd, func in available_commands.items():
epilog.add_row(" " + cmd, first_line(func.__doc__, default=""))
epilog = "Available commands:\n%s" % epilog
cls._prog = prog or package
parser = cls.parser(epilog=epilog,
help=caller.module_docstring,
prog=prog)
if cls.version and package:
parser.add_argument(*cls.version,
action="version",
version=get_version(package),
help="Show version and exit")
if cls.color:
parser.add_argument(*cls.color,
action="store_true",
help="Do not use colors (even if on tty)")
if cls.debug:
parser.add_argument(*cls.debug,
action="store_true",
help="Show debugging information")
if cls.dryrun:
parser.add_argument(*cls.dryrun,
action="store_true",
help="Perform a dryrun")
parser.add_argument("command",
choices=available_commands,
metavar="command",
help="Command to run")
parser.add_argument("args",
nargs=argparse.REMAINDER,
help="Passed-through to command")
args = parser.parse_args()
if cls.console_format or hasattr(args, "debug") or hasattr(
args, "dryrun"):
LogManager.setup(
debug=getattr(args, "debug", UNSET),
dryrun=getattr(args, "dryrun", UNSET),
console_format=cls.console_format,
console_level=cls.console_level,
default_logger=cls.default_logger,
locations=cls.log_locations,
)
color = getattr(args, "no_color", None)
if color is not None:
ColorManager.activate_colors(enable=not color)
try:
func = available_commands[args.command]
with TempArgv(args.args):
func()
except KeyboardInterrupt: # pragma: no cover
sys.stderr.write("\nAborted\n")
sys.exit(1)
def __repr__(self):
return "%s [%s]" % (self.scanner_name, short(self.location))
def _file_op(source, destination, func, overwrite, fatal, logger, dryrun, must_exist=True, ignore=None, **extra):
"""Call func(source, destination)
Args:
source (str | None): Source file or folder
destination (str | None): Destination file or folder
func (callable): Implementation function
overwrite (bool | None): True: replace existing, False: fail if destination exists, None: no destination check
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
must_exist (bool): If True, verify that source does indeed exist
ignore (callable | list | str | None): Names to be ignored
**extra: Passed-through to 'func'
Returns:
(int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed
"""
if not source or not destination or source == destination:
return 0
action = func.__name__[1:]
indicator = "<-" if action == "symlink" else "->"
description = "%s %s %s %s" % (action, short(source), indicator, short(destination))
psource = parent_folder(source)
pdest = resolved_path(destination)
if psource.startswith(pdest):
message = "Can't %s: source contained in destination" % description
return abort(message, return_value=-1, fatal=fatal, logger=logger)
if _R.hdry(dryrun, logger, description):
return 1
if must_exist and not (os.path.exists(source) or os.path.islink(source)):
message = "%s does not exist, can't %s to %s" % (short(source), action.lower(), short(destination))
return abort(message, return_value=-1, fatal=fatal, logger=logger)
if overwrite is not None:
islink = os.path.islink(pdest)
if islink or os.path.exists(pdest):
if not overwrite:
message = "%s exists, can't %s" % (short(destination), action.lower())
return abort(message, return_value=-1, fatal=fatal, logger=logger)
_do_delete(pdest, islink, fatal)
try:
# Ensure parent folder exists
ensure_folder(parent_folder(destination), fatal=fatal, logger=None, dryrun=dryrun)
_R.hlog(logger, "%s%s" % (description[0].upper(), description[1:]))
if ignore is not None:
if not callable(ignore):
given = ignore
def ignore(*_):
return given
extra["ignore"] = ignore
func(source, destination, **extra)
return 1
except Exception as e:
return abort("Can't %s" % description, exc_info=e, return_value=-1, fatal=fatal, logger=logger)
def dryrun_msg():
return "%s %s" % ("write %s to" % byte_size if byte_size else "touch", short(path))
def run(program,
*args,
background=False,
fatal=True,
logger=UNSET,
dryrun=UNSET,
short_exe=UNSET,
passthrough=False,
path_env=None,
strip="\r\n",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
**popen_args):
"""Run 'program' with 'args'
Args:
program (str | pathlib.Path): Program to run (full path, or basename)
*args: Command line args to call 'program' with
background (bool): When True, background the spawned process (detach from console and current process)
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
short_exe (str | bool | None): Try to log a compact representation of executable
passthrough (bool | file | None): If True-ish, pass-through stderr/stdout in addition to capturing it
as well as 'passthrough' itself if it has a write() function
path_env (dict | None): Allows to inject PATH-like env vars, see `_added_env_paths()`
strip (str | bool | None): If provided, `strip()` the captured output [default: strip "\n" newlines]
stdout (int | IO[Any] | None): Passed-through to subprocess.Popen, [default: subprocess.PIPE]
stderr (int | IO[Any] | None): Passed-through to subprocess.Popen, [default: subprocess.PIPE]
**popen_args: Passed through to `subprocess.Popen`
Returns:
(RunResult): Run outcome, use .failed, .succeeded, .output, .error etc to inspect the outcome
"""
if path_env:
popen_args["env"] = _added_env_paths(path_env,
env=popen_args.get("env"))
args = flattened(args, shellify=True)
full_path = which(program)
result = RunResult(audit=RunAudit(full_path or program, args, popen_args))
description = result.audit.run_description(short_exe=short_exe)
if background:
description += " &"
abort_logger = None if logger is None else UNSET
if logger is True or logger is print:
# When logger is True, we just print() the message, so we may as well color it nicely
description = _R.colored(description, "bold")
if _R.hdry(dryrun, logger, "run: %s" % description):
result.audit.dryrun = True
result.exit_code = 0
if stdout is not None:
result.output = "[dryrun] %s" % description # Properly simulate a successful run
if stdout is not None:
result.error = ""
return result
if not full_path:
if program and os.path.basename(program) == program:
result.error = "%s is not installed (PATH=%s)" % (
short(program), short(os.environ.get("PATH")))
else:
result.error = "%s is not an executable" % short(program)
return abort(result.error,
return_value=result,
fatal=fatal,
logger=abort_logger)
_R.hlog(logger, "Running: %s" % description)
if background:
child_pid = daemonize()
if child_pid:
result.pid = child_pid # In parent process, we just report a successful run (we don't wait/check on background process)
result.exit_code = 0
return result
fatal = False # pragma: no cover, non-fatal mode in background process (there is no more console etc to report anything)
with _WrappedArgs([full_path] + args) as wrapped_args:
try:
p, out, err = _run_popen(wrapped_args, popen_args, passthrough,
fatal, stdout, stderr)
result.output = decode(out, strip=strip)
result.error = decode(err, strip=strip)
result.pid = p.pid
result.exit_code = p.returncode
except Exception as e:
if fatal:
# Don't re-wrap with an abort(), let original stacktrace show through
raise
result.exc_info = e
if not result.error:
result.error = "%s failed: %s" % (
short(program), repr(e) if isinstance(e, OSError) else e)
if fatal and result.exit_code:
base_message = "%s exited with code %s" % (short(program),
result.exit_code)
if passthrough and (result.output or result.error):
exception = _R.abort_exception(override=fatal)
if exception is SystemExit:
raise SystemExit(result.exit_code)
if isinstance(exception, type) and issubclass(
exception, BaseException):
raise exception(base_message)
message = []
if abort_logger is not None and not passthrough:
# Log full output, unless user explicitly turned it off
message.append("Run failed: %s" % description)
if result.error:
message.append("\nstderr:\n%s" % result.error)
if result.output:
message.append("\nstdout:\n%s" % result.output)
message.append(base_message)
abort("\n".join(message),
code=result.exit_code,
exc_info=result.exc_info,
fatal=fatal,
logger=abort_logger)
if background:
os._exit(
result.exit_code
) # pragma: no cover, simply exit forked process (don't go back to caller)
return result
def to_path(path, no_spaces=False):
"""
Args:
path (str | Path): Path to convert
no_spaces (type | bool | None): If True-ish, abort if 'path' contains a space
Returns:
(Path | None): Converted to `Path` object, if necessary
"""
if no_spaces and " " in str(path):
abort("Refusing path with space (not worth escaping all the things to make this work): '%s'" % short(path), fatal=no_spaces)
if isinstance(path, Path):
return path
if path is not None:
if path:
path = os.path.expanduser(path)
return Path(path)
def __str__(self):
text = _R.colored(self.problem, "red") or self.spec
text = joined(text,
self.is_invoker and _R.colored("invoker", "green"),
delimiter=", ")
return "%s [%s]" % (_R.colored(short(self.short_name), "bold"), text)
