def words(text, normalize=None, split="_", decamel=False): """Words extracted from `text` (split on underscore character as well by default) Args: text: Text to extract words from normalize (callable | None): Optional function to apply on each word split (str | None): Optional extra character to split words on decamel (bool): If True, extract CamelCase words as well Returns: (list): Extracted words """ if not text: return [] if isinstance(text, list): result = [] for line in text: result.extend(words(line, normalize=normalize, split=split, decamel=decamel)) return result strings = _R.lc.rx_words.split(stringified(text)) strings = flattened(strings, split=split, strip=True) if decamel: strings = flattened(_R.lc.rx_camel_cased_words.findall(s) for s in strings) if normalize: strings = [normalize(s) for s in strings] return strings
def scan_path_env_var(self): """Ensure env vars locations are scanned Returns: (list[PythonInstallation] | None): Installations """ if self.from_path is not None: return None self.from_path = [] found = [] real_paths = defaultdict(list) for folder in flattened(os.environ.get("PATH"), split=os.pathsep): for path in self.python_exes_in_folder(folder): real_path = os.path.realpath(path) if real_path not in self._cache: real_paths[real_path].append(path) if path != real_path: real_paths[real_path].append(real_path) for real_path, paths in real_paths.items(): python = self._python_from_path(paths[0], equivalents=paths) if python.problem: _R.hlog(self.logger, "Ignoring invalid python in PATH: %s" % paths[0]) else: self._register(python, self.from_path) found.append(python) _R.hlog(self.logger, "Found %s pythons in $PATH" % (len(found))) return found
def filesize(*paths, logger=False): """ Args: *paths (str | Path | None): Paths to files/folders logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter Returns: (int): File size in bytes """ size = 0 for path in flattened(paths, unique=True): path = to_path(path) if path and path.exists() and not path.is_symlink(): if path.is_dir(): for sf in path.iterdir(): size += filesize(sf) elif path.is_file(): try: size += path.stat().st_size except Exception as e: # pragma: no cover, ignore cases like permission denied, file name too long, etc _R.hlog(logger, "Can't stat %s: %s" % (short(path), short(e, size=32))) return size
def find_preferred_python(self, preferred_pythons, min_python="3.6", preferred_min_python="3.7"): """First python in 'preferred_pythons' that satisfies the stated minimum versions Args: preferred_pythons (str | list): Comma-separated list of desired preferred pythons min_python (str): Minimum version to consider (as 2nd best) preferred_min_python (str): Minimum version to consider, if available """ self.preferred_python = None if preferred_pythons: second_best = None for pref in flattened(preferred_pythons, split=","): if pref: python = self._find_python(pref, False, False) if python and not python.problem: if preferred_min_python and python.version >= preferred_min_python: self.preferred_python = python return if min_python and not second_best and python.version >= min_python: second_best = python self.preferred_python = second_best
def __init__(self, frames, fps=10): """ Args: frames: Frames composing the ascii animation fps (int): Desired frames per second """ self.frames = flattened(frames) or None self.fps = fps self.index = 0
def _get_values(self, value): value = flattened(value, split=self.split) values = [t.partition("=") for t in value if t] values = dict((k, v) for k, _, v in values) if self.prefix: values = dict( (affixed(k, prefix=self.prefix), v) for k, v in values.items()) return values
def speccified(cls, values, strict=False): """ Args: values (Iterable | None): Values to transform into a list of PythonSpec-s Returns: (list[PythonSpec]): Corresponding list of PythonSpec-s """ values = flattened(values, split=",", transform=PythonSpec.to_spec) if strict: values = [x for x in values if x.version] return values
def is_using_format(cls, markers, used_formats=None): """ Args: markers (str): Space separated list of markers to look for used_formats (str): Formats to consider (default: cls.used_formats) Returns: (bool): True if any one of the 'markers' is seen in 'used_formats' """ if used_formats is None: used_formats = cls.used_formats if not markers or not used_formats: return False return any(marker in used_formats for marker in flattened(markers, split=" "))
def settings(help=None, width=140, **attrs): """ Args: help (list[str] | str | None): List of flags to show help, default: -h and --help width (int): Constrain help to **attrs: Returns: dict: Dict passable to @click.command() or @click.group() """ if help is None: help = ["-h", "--help"] context_settings = attrs.pop("context_settings", {}) context_settings["help_option_names"] = flattened(help, split=" ") context_settings["max_content_width"] = width return dict(context_settings=context_settings, **attrs)
def __init__(self, value=None): self._columns = [] self.shown = True if value is None: return if isinstance(value, str): for t in flattened(value, split=",", keep_empty=True): self.add_column(t) elif isinstance(value, int): self.accommodate(value) elif hasattr(value, "__iter__"): for x in value: self.add_column(x) else: raise ValueError("Invalid header '%s'" % value)
def run(self, *args, exe=None, main=UNSET, trace=UNSET): """ Args: *args: Command line args exe (str | None): Optional, override sys.argv[0] just for this run main (callable | None): Optional, override current self.main just for this run trace (bool): If True, enable trace logging """ main = _R.rdefault(main, self.main or cli.default_main) if len(args) == 1 and hasattr(args[0], "split"): # Convenience: allow to provide full command as one string argument args = args[0].split() self.args = flattened(args, shellify=True) with IsolatedLogSetup(adjust_tmp=False): with CaptureOutput(dryrun=_R.is_dryrun(), seed_logging=True, trace=_R.rdefault(trace, self.trace)) as logged: self.logged = logged with TempArgv(self.args, exe=exe): result = self._run_main(main, self.args) if isinstance(result.exception, AssertionError): raise result.exception if result.stdout: logged.stdout.buffer.write(result.stdout) if result.stderr: logged.stderr.buffer.write(result.stderr) if result.exception and not isinstance(result.exception, SystemExit): try: raise result.exception except Exception: LOG.exception("Exited with stacktrace:") self.exit_code = result.exit_code if self.logged: WrappedHandler.clean_accumulated_logs() title = Header.aerated("Captured output for: %s" % quoted(self.args), border="==") LOG.info("\n%s\nmain: %s\nexit_code: %s\n%s\n", title, main, self.exit_code, self.logged)
def add_row(self, *values): """Add one row with given 'values'""" row = flattened(values, keep_empty=True) self.header.accommodate(len(row)) self._rows.append(row)
def setup( cls, debug=UNSET, dryrun=UNSET, level=UNSET, clean_handlers=UNSET, greetings=UNSET, appname=UNSET, basename=UNSET, console_format=UNSET, console_level=UNSET, console_stream=UNSET, context_format=UNSET, default_logger=UNSET, dev=UNSET, file_format=UNSET, file_level=UNSET, file_location=UNSET, locations=UNSET, rotate=UNSET, rotate_count=UNSET, timezone=UNSET, tmp=UNSET, trace=UNSET, allow_root=UNSET, ): """ Args: debug (bool): Enable debug level logging (overrides other specified levels) dryrun (bool): Enable dryrun level (int | None): Shortcut to set both `console_level` and `file_level` at once clean_handlers (bool): Remove any existing logging.root.handlers greetings (str | None): Optional greetings message(s) to log appname (str | None): Program's base name, not used directly, just as reference for default 'basename' basename (str | None): Base name of target log file, not used directly, just as reference for default 'locations' console_format (str | None): Format to use for console log, use None to deactivate console_level (int | None): Level to use for console logging console_stream (io.TextIOBase | TextIO | None): Stream to use for console log (eg: sys.stderr), use None to deactivate context_format (str | None): Format to use for contextual log, use None to deactivate default_logger (callable | None): Default logger to use to trace operations such as runez.run() etc dev (str | None): Custom folder to use when running from a development venv (auto-determined if None) file_format (str | None): Format to use for file log, use None to deactivate file_level (int | None): Level to use for file logging file_location (str | None): Desired custom file location (overrides {locations} search, handy as a --log cli flag) locations (list[str]|None): List of candidate folders for file logging (None: deactivate file logging) rotate (str | None): How to rotate log file (None: no rotation, "time:1d" time-based, "size:50m" size-based) rotate_count (int): How many rotations to keep timezone (str | None): Time zone, use None to deactivate time zone logging tmp (str | None): Optional temp folder to use (auto determined) trace (str | bool): Env var to enable tracing, example: "DEBUG+| " to trace when $DEBUG defined (+ [optional] "| " as prefix) allow_root (bool | None): True allows running as root, None aborts execution if ran as root (default: allowed in docker only) """ with cls._lock: cls.set_debug(debug) cls.set_dryrun(dryrun) cls.spec.set( appname=appname, basename=basename, console_format=console_format, console_level=console_level or level, console_stream=console_stream, context_format=context_format, default_logger=default_logger, dev=dev, file_format=file_format, file_level=file_level or level, file_location=file_location, locations=locations, rotate=rotate, rotate_count=rotate_count, timezone=timezone, tmp=tmp, ) cls._auto_fill_defaults() if cls.debug: cls.spec.console_level = logging.DEBUG cls.spec.file_level = logging.DEBUG elif level: cls.spec.console_level = level cls.spec.file_level = level root_level = min( flattened(cls.spec.console_level, cls.spec.file_level)) if root_level and root_level != logging.root.level: logging.root.setLevel(root_level) if trace is UNSET: if cls.tracer is None: trace = cls.trace_env_var elif isinstance(trace, str): cls.trace_env_var = trace if isinstance(trace, str) and "+" in trace: p = trace.partition("+") cls.enable_trace(p[0], prefix=p[2]) else: cls.enable_trace(trace) if cls.handlers is None: cls.handlers = [] cls._setup_console_handler() cls._setup_file_handler() cls._auto_enable_progress_handler() cls._update_used_formats() cls._fix_logging_shortcuts() if clean_handlers: cls.clean_handlers() cls.greet(greetings) if allow_root is UNSET: allow_root = SYS_INFO.is_running_in_docker if not allow_root and os.geteuid() == 0: message = _formatted_text(cls.disallow_root_message, cls._props(), strict=False) if message.endswith("!"): bars = "=" * len(message) message = "\n%s\n%s\n%s\n\n" % (bars, message, bars) message = _R.colored(message, "red") abort_if(allow_root is None, message) LOG.warning(message)
def run(program, *args, background=False, fatal=True, logger=UNSET, dryrun=UNSET, short_exe=UNSET, passthrough=False, path_env=None, strip="\r\n", stdout=subprocess.PIPE, stderr=subprocess.PIPE, **popen_args): """Run 'program' with 'args' Args: program (str | pathlib.Path): Program to run (full path, or basename) *args: Command line args to call 'program' with background (bool): When True, background the spawned process (detach from console and current process) fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter dryrun (bool | UNSET | None): Optionally override current dryrun setting short_exe (str | bool | None): Try to log a compact representation of executable passthrough (bool | file | None): If True-ish, pass-through stderr/stdout in addition to capturing it as well as 'passthrough' itself if it has a write() function path_env (dict | None): Allows to inject PATH-like env vars, see `_added_env_paths()` strip (str | bool | None): If provided, `strip()` the captured output [default: strip "\n" newlines] stdout (int | IO[Any] | None): Passed-through to subprocess.Popen, [default: subprocess.PIPE] stderr (int | IO[Any] | None): Passed-through to subprocess.Popen, [default: subprocess.PIPE] **popen_args: Passed through to `subprocess.Popen` Returns: (RunResult): Run outcome, use .failed, .succeeded, .output, .error etc to inspect the outcome """ if path_env: popen_args["env"] = _added_env_paths(path_env, env=popen_args.get("env")) args = flattened(args, shellify=True) full_path = which(program) result = RunResult(audit=RunAudit(full_path or program, args, popen_args)) description = result.audit.run_description(short_exe=short_exe) if background: description += " &" abort_logger = None if logger is None else UNSET if logger is True or logger is print: # When logger is True, we just print() the message, so we may as well color it nicely description = _R.colored(description, "bold") if _R.hdry(dryrun, logger, "run: %s" % description): result.audit.dryrun = True result.exit_code = 0 if stdout is not None: result.output = "[dryrun] %s" % description # Properly simulate a successful run if stdout is not None: result.error = "" return result if not full_path: if program and os.path.basename(program) == program: result.error = "%s is not installed (PATH=%s)" % ( short(program), short(os.environ.get("PATH"))) else: result.error = "%s is not an executable" % short(program) return abort(result.error, return_value=result, fatal=fatal, logger=abort_logger) _R.hlog(logger, "Running: %s" % description) if background: child_pid = daemonize() if child_pid: result.pid = child_pid # In parent process, we just report a successful run (we don't wait/check on background process) result.exit_code = 0 return result fatal = False # pragma: no cover, non-fatal mode in background process (there is no more console etc to report anything) with _WrappedArgs([full_path] + args) as wrapped_args: try: p, out, err = _run_popen(wrapped_args, popen_args, passthrough, fatal, stdout, stderr) result.output = decode(out, strip=strip) result.error = decode(err, strip=strip) result.pid = p.pid result.exit_code = p.returncode except Exception as e: if fatal: # Don't re-wrap with an abort(), let original stacktrace show through raise result.exc_info = e if not result.error: result.error = "%s failed: %s" % ( short(program), repr(e) if isinstance(e, OSError) else e) if fatal and result.exit_code: base_message = "%s exited with code %s" % (short(program), result.exit_code) if passthrough and (result.output or result.error): exception = _R.abort_exception(override=fatal) if exception is SystemExit: raise SystemExit(result.exit_code) if isinstance(exception, type) and issubclass( exception, BaseException): raise exception(base_message) message = [] if abort_logger is not None and not passthrough: # Log full output, unless user explicitly turned it off message.append("Run failed: %s" % description) if result.error: message.append("\nstderr:\n%s" % result.error) if result.output: message.append("\nstdout:\n%s" % result.output) message.append(base_message) abort("\n".join(message), code=result.exit_code, exc_info=result.exc_info, fatal=fatal, logger=abort_logger) if background: os._exit( result.exit_code ) # pragma: no cover, simply exit forked process (don't go back to caller) return result
def auto_shellify(args): if args and len(args) == 1 and hasattr(args[0], "split"): return args[0].split() return flattened(args, shellify=True)