def words(text, normalize=None, split="_", decamel=False):
"""Words extracted from `text` (split on underscore character as well by default)
Args:
text: Text to extract words from
normalize (callable | None): Optional function to apply on each word
split (str | None): Optional extra character to split words on
decamel (bool): If True, extract CamelCase words as well
Returns:
(list): Extracted words
"""
if not text:
return []
if isinstance(text, list):
result = []
for line in text:
result.extend(words(line, normalize=normalize, split=split, decamel=decamel))
return result
strings = _R.lc.rx_words.split(stringified(text))
strings = flattened(strings, split=split, strip=True)
if decamel:
strings = flattened(_R.lc.rx_camel_cased_words.findall(s) for s in strings)
if normalize:
strings = [normalize(s) for s in strings]
return strings
def scan_path_env_var(self):
"""Ensure env vars locations are scanned
Returns:
(list[PythonInstallation] | None): Installations
"""
if self.from_path is not None:
return None
self.from_path = []
found = []
real_paths = defaultdict(list)
for folder in flattened(os.environ.get("PATH"), split=os.pathsep):
for path in self.python_exes_in_folder(folder):
real_path = os.path.realpath(path)
if real_path not in self._cache:
real_paths[real_path].append(path)
if path != real_path:
real_paths[real_path].append(real_path)
for real_path, paths in real_paths.items():
python = self._python_from_path(paths[0], equivalents=paths)
if python.problem:
_R.hlog(self.logger,
"Ignoring invalid python in PATH: %s" % paths[0])
else:
self._register(python, self.from_path)
found.append(python)
_R.hlog(self.logger, "Found %s pythons in $PATH" % (len(found)))
return found
def filesize(*paths, logger=False):
"""
Args:
*paths (str | Path | None): Paths to files/folders
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
Returns:
(int): File size in bytes
"""
size = 0
for path in flattened(paths, unique=True):
path = to_path(path)
if path and path.exists() and not path.is_symlink():
if path.is_dir():
for sf in path.iterdir():
size += filesize(sf)
elif path.is_file():
try:
size += path.stat().st_size
except Exception as e: # pragma: no cover, ignore cases like permission denied, file name too long, etc
_R.hlog(logger, "Can't stat %s: %s" % (short(path), short(e, size=32)))
return size
def find_preferred_python(self,
preferred_pythons,
min_python="3.6",
preferred_min_python="3.7"):
"""First python in 'preferred_pythons' that satisfies the stated minimum versions
Args:
preferred_pythons (str | list): Comma-separated list of desired preferred pythons
min_python (str): Minimum version to consider (as 2nd best)
preferred_min_python (str): Minimum version to consider, if available
"""
self.preferred_python = None
if preferred_pythons:
second_best = None
for pref in flattened(preferred_pythons, split=","):
if pref:
python = self._find_python(pref, False, False)
if python and not python.problem:
if preferred_min_python and python.version >= preferred_min_python:
self.preferred_python = python
return
if min_python and not second_best and python.version >= min_python:
second_best = python
self.preferred_python = second_best
def __init__(self, frames, fps=10):
"""
Args:
frames: Frames composing the ascii animation
fps (int): Desired frames per second
"""
self.frames = flattened(frames) or None
self.fps = fps
self.index = 0
def _get_values(self, value):
value = flattened(value, split=self.split)
values = [t.partition("=") for t in value if t]
values = dict((k, v) for k, _, v in values)
if self.prefix:
values = dict(
(affixed(k, prefix=self.prefix), v) for k, v in values.items())
return values
def speccified(cls, values, strict=False):
"""
Args:
values (Iterable | None): Values to transform into a list of PythonSpec-s
Returns:
(list[PythonSpec]): Corresponding list of PythonSpec-s
"""
values = flattened(values, split=",", transform=PythonSpec.to_spec)
if strict:
values = [x for x in values if x.version]
return values
def is_using_format(cls, markers, used_formats=None):
"""
Args:
markers (str): Space separated list of markers to look for
used_formats (str): Formats to consider (default: cls.used_formats)
Returns:
(bool): True if any one of the 'markers' is seen in 'used_formats'
"""
if used_formats is None:
used_formats = cls.used_formats
if not markers or not used_formats:
return False
return any(marker in used_formats
for marker in flattened(markers, split=" "))
def settings(help=None, width=140, **attrs):
"""
Args:
help (list[str] | str | None): List of flags to show help, default: -h and --help
width (int): Constrain help to
**attrs:
Returns:
dict: Dict passable to @click.command() or @click.group()
"""
if help is None:
help = ["-h", "--help"]
context_settings = attrs.pop("context_settings", {})
context_settings["help_option_names"] = flattened(help, split=" ")
context_settings["max_content_width"] = width
return dict(context_settings=context_settings, **attrs)
def __init__(self, value=None):
self._columns = []
self.shown = True
if value is None:
return
if isinstance(value, str):
for t in flattened(value, split=",", keep_empty=True):
self.add_column(t)
elif isinstance(value, int):
self.accommodate(value)
elif hasattr(value, "__iter__"):
for x in value:
self.add_column(x)
else:
raise ValueError("Invalid header '%s'" % value)
def run(self, *args, exe=None, main=UNSET, trace=UNSET):
"""
Args:
*args: Command line args
exe (str | None): Optional, override sys.argv[0] just for this run
main (callable | None): Optional, override current self.main just for this run
trace (bool): If True, enable trace logging
"""
main = _R.rdefault(main, self.main or cli.default_main)
if len(args) == 1 and hasattr(args[0], "split"):
# Convenience: allow to provide full command as one string argument
args = args[0].split()
self.args = flattened(args, shellify=True)
with IsolatedLogSetup(adjust_tmp=False):
with CaptureOutput(dryrun=_R.is_dryrun(), seed_logging=True, trace=_R.rdefault(trace, self.trace)) as logged:
self.logged = logged
with TempArgv(self.args, exe=exe):
result = self._run_main(main, self.args)
if isinstance(result.exception, AssertionError):
raise result.exception
if result.stdout:
logged.stdout.buffer.write(result.stdout)
if result.stderr:
logged.stderr.buffer.write(result.stderr)
if result.exception and not isinstance(result.exception, SystemExit):
try:
raise result.exception
except Exception:
LOG.exception("Exited with stacktrace:")
self.exit_code = result.exit_code
if self.logged:
WrappedHandler.clean_accumulated_logs()
title = Header.aerated("Captured output for: %s" % quoted(self.args), border="==")
LOG.info("\n%s\nmain: %s\nexit_code: %s\n%s\n", title, main, self.exit_code, self.logged)
def add_row(self, *values):
"""Add one row with given 'values'"""
row = flattened(values, keep_empty=True)
self.header.accommodate(len(row))
self._rows.append(row)
def setup(
cls,
debug=UNSET,
dryrun=UNSET,
level=UNSET,
clean_handlers=UNSET,
greetings=UNSET,
appname=UNSET,
basename=UNSET,
console_format=UNSET,
console_level=UNSET,
console_stream=UNSET,
context_format=UNSET,
default_logger=UNSET,
dev=UNSET,
file_format=UNSET,
file_level=UNSET,
file_location=UNSET,
locations=UNSET,
rotate=UNSET,
rotate_count=UNSET,
timezone=UNSET,
tmp=UNSET,
trace=UNSET,
allow_root=UNSET,
):
"""
Args:
debug (bool): Enable debug level logging (overrides other specified levels)
dryrun (bool): Enable dryrun
level (int | None): Shortcut to set both `console_level` and `file_level` at once
clean_handlers (bool): Remove any existing logging.root.handlers
greetings (str | None): Optional greetings message(s) to log
appname (str | None): Program's base name, not used directly, just as reference for default 'basename'
basename (str | None): Base name of target log file, not used directly, just as reference for default 'locations'
console_format (str | None): Format to use for console log, use None to deactivate
console_level (int | None): Level to use for console logging
console_stream (io.TextIOBase | TextIO | None): Stream to use for console log (eg: sys.stderr), use None to deactivate
context_format (str | None): Format to use for contextual log, use None to deactivate
default_logger (callable | None): Default logger to use to trace operations such as runez.run() etc
dev (str | None): Custom folder to use when running from a development venv (auto-determined if None)
file_format (str | None): Format to use for file log, use None to deactivate
file_level (int | None): Level to use for file logging
file_location (str | None): Desired custom file location (overrides {locations} search, handy as a --log cli flag)
locations (list[str]|None): List of candidate folders for file logging (None: deactivate file logging)
rotate (str | None): How to rotate log file (None: no rotation, "time:1d" time-based, "size:50m" size-based)
rotate_count (int): How many rotations to keep
timezone (str | None): Time zone, use None to deactivate time zone logging
tmp (str | None): Optional temp folder to use (auto determined)
trace (str | bool): Env var to enable tracing, example: "DEBUG+| " to trace when $DEBUG defined (+ [optional] "| " as prefix)
allow_root (bool | None): True allows running as root, None aborts execution if ran as root (default: allowed in docker only)
"""
with cls._lock:
cls.set_debug(debug)
cls.set_dryrun(dryrun)
cls.spec.set(
appname=appname,
basename=basename,
console_format=console_format,
console_level=console_level or level,
console_stream=console_stream,
context_format=context_format,
default_logger=default_logger,
dev=dev,
file_format=file_format,
file_level=file_level or level,
file_location=file_location,
locations=locations,
rotate=rotate,
rotate_count=rotate_count,
timezone=timezone,
tmp=tmp,
)
cls._auto_fill_defaults()
if cls.debug:
cls.spec.console_level = logging.DEBUG
cls.spec.file_level = logging.DEBUG
elif level:
cls.spec.console_level = level
cls.spec.file_level = level
root_level = min(
flattened(cls.spec.console_level, cls.spec.file_level))
if root_level and root_level != logging.root.level:
logging.root.setLevel(root_level)
if trace is UNSET:
if cls.tracer is None:
trace = cls.trace_env_var
elif isinstance(trace, str):
cls.trace_env_var = trace
if isinstance(trace, str) and "+" in trace:
p = trace.partition("+")
cls.enable_trace(p[0], prefix=p[2])
else:
cls.enable_trace(trace)
if cls.handlers is None:
cls.handlers = []
cls._setup_console_handler()
cls._setup_file_handler()
cls._auto_enable_progress_handler()
cls._update_used_formats()
cls._fix_logging_shortcuts()
if clean_handlers:
cls.clean_handlers()
cls.greet(greetings)
if allow_root is UNSET:
allow_root = SYS_INFO.is_running_in_docker
if not allow_root and os.geteuid() == 0:
message = _formatted_text(cls.disallow_root_message,
cls._props(),
strict=False)
if message.endswith("!"):
bars = "=" * len(message)
message = "\n%s\n%s\n%s\n\n" % (bars, message, bars)
message = _R.colored(message, "red")
abort_if(allow_root is None, message)
LOG.warning(message)
def run(program,
*args,
background=False,
fatal=True,
logger=UNSET,
dryrun=UNSET,
short_exe=UNSET,
passthrough=False,
path_env=None,
strip="\r\n",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
**popen_args):
"""Run 'program' with 'args'
Args:
program (str | pathlib.Path): Program to run (full path, or basename)
*args: Command line args to call 'program' with
background (bool): When True, background the spawned process (detach from console and current process)
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
short_exe (str | bool | None): Try to log a compact representation of executable
passthrough (bool | file | None): If True-ish, pass-through stderr/stdout in addition to capturing it
as well as 'passthrough' itself if it has a write() function
path_env (dict | None): Allows to inject PATH-like env vars, see `_added_env_paths()`
strip (str | bool | None): If provided, `strip()` the captured output [default: strip "\n" newlines]
stdout (int | IO[Any] | None): Passed-through to subprocess.Popen, [default: subprocess.PIPE]
stderr (int | IO[Any] | None): Passed-through to subprocess.Popen, [default: subprocess.PIPE]
**popen_args: Passed through to `subprocess.Popen`
Returns:
(RunResult): Run outcome, use .failed, .succeeded, .output, .error etc to inspect the outcome
"""
if path_env:
popen_args["env"] = _added_env_paths(path_env,
env=popen_args.get("env"))
args = flattened(args, shellify=True)
full_path = which(program)
result = RunResult(audit=RunAudit(full_path or program, args, popen_args))
description = result.audit.run_description(short_exe=short_exe)
if background:
description += " &"
abort_logger = None if logger is None else UNSET
if logger is True or logger is print:
# When logger is True, we just print() the message, so we may as well color it nicely
description = _R.colored(description, "bold")
if _R.hdry(dryrun, logger, "run: %s" % description):
result.audit.dryrun = True
result.exit_code = 0
if stdout is not None:
result.output = "[dryrun] %s" % description # Properly simulate a successful run
if stdout is not None:
result.error = ""
return result
if not full_path:
if program and os.path.basename(program) == program:
result.error = "%s is not installed (PATH=%s)" % (
short(program), short(os.environ.get("PATH")))
else:
result.error = "%s is not an executable" % short(program)
return abort(result.error,
return_value=result,
fatal=fatal,
logger=abort_logger)
_R.hlog(logger, "Running: %s" % description)
if background:
child_pid = daemonize()
if child_pid:
result.pid = child_pid # In parent process, we just report a successful run (we don't wait/check on background process)
result.exit_code = 0
return result
fatal = False # pragma: no cover, non-fatal mode in background process (there is no more console etc to report anything)
with _WrappedArgs([full_path] + args) as wrapped_args:
try:
p, out, err = _run_popen(wrapped_args, popen_args, passthrough,
fatal, stdout, stderr)
result.output = decode(out, strip=strip)
result.error = decode(err, strip=strip)
result.pid = p.pid
result.exit_code = p.returncode
except Exception as e:
if fatal:
# Don't re-wrap with an abort(), let original stacktrace show through
raise
result.exc_info = e
if not result.error:
result.error = "%s failed: %s" % (
short(program), repr(e) if isinstance(e, OSError) else e)
if fatal and result.exit_code:
base_message = "%s exited with code %s" % (short(program),
result.exit_code)
if passthrough and (result.output or result.error):
exception = _R.abort_exception(override=fatal)
if exception is SystemExit:
raise SystemExit(result.exit_code)
if isinstance(exception, type) and issubclass(
exception, BaseException):
raise exception(base_message)
message = []
if abort_logger is not None and not passthrough:
# Log full output, unless user explicitly turned it off
message.append("Run failed: %s" % description)
if result.error:
message.append("\nstderr:\n%s" % result.error)
if result.output:
message.append("\nstdout:\n%s" % result.output)
message.append(base_message)
abort("\n".join(message),
code=result.exit_code,
exc_info=result.exc_info,
fatal=fatal,
logger=abort_logger)
if background:
os._exit(
result.exit_code
) # pragma: no cover, simply exit forked process (don't go back to caller)
return result
def auto_shellify(args):
if args and len(args) == 1 and hasattr(args[0], "split"):
return args[0].split()
return flattened(args, shellify=True)