def readlines(path, first=None, errors="ignore", fatal=False, logger=False):
"""
Args:
path (str | Path | None): Path to file to read lines from
first (int | None): Return only the 'first' lines when specified
errors (str | None): Optional string specifying how encoding errors are to be handled
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
Yields:
(str): Lines read, newlines and trailing spaces stripped
"""
try:
with io.open(resolved_path(path), errors=errors) as fh:
if not first:
first = -1
for line in fh:
if first == 0:
return
yield decode(line).rstrip()
first -= 1
except Exception as e:
message = "Can't read %s" % short(path)
if fatal:
abort(_R.actual_message(message), exc_info=e, fatal=fatal, logger=logger)
_R.hlog(logger, message, exc_info=e)
def delete(path, fatal=True, logger=UNSET, dryrun=UNSET):
"""
Args:
path (str | Path | None): Path to file or folder to delete
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
Returns:
(int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed
"""
path = resolved_path(path)
islink = path and os.path.islink(path)
if not islink and (not path or not os.path.exists(path)):
return 0
if _R.hdry(dryrun, logger, "delete %s" % short(path)):
return 1
try:
_do_delete(path, islink, fatal)
_R.hlog(logger, "Deleted %s" % short(path))
return 1
except Exception as e:
return abort("Can't delete %s" % short(path), exc_info=e, return_value=-1, fatal=fatal, logger=logger)
def make_executable(path, fatal=True, logger=UNSET, dryrun=UNSET):
"""
Args:
path (str): chmod file with 'path' as executable
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
Returns:
(int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed
"""
if is_executable(path):
return 0
if _R.hdry(dryrun, logger, "make %s executable" % short(path)):
return 1
if not os.path.exists(path):
return abort("%s does not exist, can't make it executable" %
short(path),
return_value=-1,
fatal=fatal,
logger=logger)
try:
os.chmod(path, 0o755) # nosec
_R.hlog(logger, "Made '%s' executable" % short(path))
return 1
except Exception as e:
return abort("Can't chmod %s" % short(path),
exc_info=e,
return_value=-1,
fatal=fatal,
logger=logger)
def filesize(*paths, logger=False):
"""
Args:
*paths (str | Path | None): Paths to files/folders
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
Returns:
(int): File size in bytes
"""
size = 0
for path in flattened(paths, unique=True):
path = to_path(path)
if path and path.exists() and not path.is_symlink():
if path.is_dir():
for sf in path.iterdir():
size += filesize(sf)
elif path.is_file():
try:
size += path.stat().st_size
except Exception as e: # pragma: no cover, ignore cases like permission denied, file name too long, etc
_R.hlog(logger, "Can't stat %s: %s" % (short(path), short(e, size=32)))
return size
def scan_path_env_var(self):
"""Ensure env vars locations are scanned
Returns:
(list[PythonInstallation] | None): Installations
"""
if self.from_path is not None:
return None
self.from_path = []
found = []
real_paths = defaultdict(list)
for folder in flattened(os.environ.get("PATH"), split=os.pathsep):
for path in self.python_exes_in_folder(folder):
real_path = os.path.realpath(path)
if real_path not in self._cache:
real_paths[real_path].append(path)
if path != real_path:
real_paths[real_path].append(real_path)
for real_path, paths in real_paths.items():
python = self._python_from_path(paths[0], equivalents=paths)
if python.problem:
_R.hlog(self.logger,
"Ignoring invalid python in PATH: %s" % paths[0])
else:
self._register(python, self.from_path)
found.append(python)
_R.hlog(self.logger, "Found %s pythons in $PATH" % (len(found)))
return found
def _get_response(self, method, url, fatal, logger, dryrun=False, state=None, action=None, **kwargs):
"""
Args:
method (str): Underlying method to call
url (str): Remote URL (may be absolute, or relative to self.base_url)
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
state (DataState | None): For PUT/POST requests
action (str | None): Action to refer to in dryrun message (default: method)
**kwargs: Passed through to underlying client
Returns:
(RestResponse): Response from underlying call
"""
absolute_url = self.full_url(url)
message = "%s %s" % (action or method, absolute_url)
if _R.hdry(dryrun, logger, message):
return RestResponse(method, absolute_url, MockResponse(200, dict(message="dryrun %s" % message)))
full_headers = self.headers
headers = kwargs.get("headers")
if headers:
full_headers = dict(full_headers)
full_headers.update(headers)
keyword_args = dict(kwargs)
keyword_args["headers"] = full_headers
keyword_args.setdefault("timeout", self.timeout)
if state is not None:
state.complete(keyword_args)
try:
raw_response = self._protected_get(method, absolute_url, keyword_args)
response = self.handler.to_rest_response(method, absolute_url, raw_response)
if fatal or logger is not None:
msg = response.description()
if fatal and not response.ok:
abort(msg, fatal=fatal, logger=logger)
_R.hlog(logger, msg)
return response
finally:
if state is not None:
state.close()
def ensure_folder(path, clean=False, fatal=True, logger=UNSET, dryrun=UNSET):
"""Ensure folder with 'path' exists
Args:
path (str | Path | None): Path to file or folder
clean (bool): True: If True, ensure folder is clean (delete any file/folder it may have)
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
Returns:
(int): In non-fatal mode, >=1: successfully done, 0: was no-op, -1: failed
"""
path = resolved_path(path)
if not path:
return 0
if os.path.isdir(path):
if not clean:
return 0
cleaned = 0
for fname in os.listdir(path):
cleaned += delete(os.path.join(path, fname), fatal=fatal, logger=None, dryrun=dryrun)
if cleaned:
msg = "%s from %s" % (_R.lc.rm.plural(cleaned, "file"), short(path))
if not _R.hdry(dryrun, logger, "clean %s" % msg):
_R.hlog(logger, "Cleaned %s" % msg)
return cleaned
if _R.hdry(dryrun, logger, "create %s" % short(path)):
return 1
try:
os.makedirs(path)
_R.hlog(logger, "Created folder %s" % short(path))
return 1
except Exception as e:
return abort("Can't create folder %s" % short(path), exc_info=e, return_value=-1, fatal=fatal, logger=logger)
def __init__(self, scanner=None, use_path=UNSET, logger=False):
"""
Args:
scanner (PythonInstallationScanner | None): Optional additional scanner to use
use_path (bool): Scan $PATH env var? (default: class attribute default)
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
"""
self.py_scanner = scanner
if use_path is not UNSET:
self.use_path = use_path
self.logger = logger
self.from_path = None if self.use_path else []
self._cache = {}
self.scanned = []
scanned = scanner.scan() if scanner else None
if scanned:
for python in scanned:
if python:
if python.problem:
_R.hlog(
self.logger, "Ignoring invalid python in %s: %s" %
(self, python))
else:
self._register(python, self.scanned)
self.scanned = sorted(self.scanned, reverse=True)
_R.hlog(self.logger,
"Found %s pythons in %s" % (len(self.scanned), scanner))
self.invoker = self._cache.get(sys.base_prefix)
if self.use_path and self.invoker is None:
self.scan_path_env_var()
self.invoker = self._cache.get(sys.base_prefix)
if self.invoker is None:
self.invoker = self._find_invoker()
self.invoker.is_invoker = True
self._cache["invoker"] = self.invoker
def write(path, contents, fatal=True, logger=UNSET, dryrun=UNSET):
"""Write `contents` to file with `path`
Args:
path (str | Path | None): Path to file
contents (str | bytes | None): Contents to write (only touch file if None)
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
Returns:
(int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed
"""
if not path:
return 0
path = resolved_path(path)
byte_size = _R.lc.rm.represented_bytesize(len(contents), unit="bytes") if contents else ""
def dryrun_msg():
return "%s %s" % ("write %s to" % byte_size if byte_size else "touch", short(path))
if _R.hdry(dryrun, logger, dryrun_msg):
return 1
ensure_folder(parent_folder(path), fatal=fatal, logger=None, dryrun=dryrun)
try:
mode = "wb" if isinstance(contents, bytes) else "wt"
with io.open(path, mode) as fh:
if contents is None:
os.utime(path, None)
else:
fh.write(contents)
_R.hlog(logger, "%s %s" % ("Wrote %s to" % byte_size if byte_size else "Touched", short(path)))
return 1
except Exception as e:
return abort("Can't write to %s" % short(path), exc_info=e, return_value=-1, fatal=fatal, logger=logger)
def _file_op(source, destination, func, overwrite, fatal, logger, dryrun, must_exist=True, ignore=None, **extra):
"""Call func(source, destination)
Args:
source (str | None): Source file or folder
destination (str | None): Destination file or folder
func (callable): Implementation function
overwrite (bool | None): True: replace existing, False: fail if destination exists, None: no destination check
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
must_exist (bool): If True, verify that source does indeed exist
ignore (callable | list | str | None): Names to be ignored
**extra: Passed-through to 'func'
Returns:
(int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed
"""
if not source or not destination or source == destination:
return 0
action = func.__name__[1:]
indicator = "<-" if action == "symlink" else "->"
description = "%s %s %s %s" % (action, short(source), indicator, short(destination))
psource = parent_folder(source)
pdest = resolved_path(destination)
if psource.startswith(pdest):
message = "Can't %s: source contained in destination" % description
return abort(message, return_value=-1, fatal=fatal, logger=logger)
if _R.hdry(dryrun, logger, description):
return 1
if must_exist and not (os.path.exists(source) or os.path.islink(source)):
message = "%s does not exist, can't %s to %s" % (short(source), action.lower(), short(destination))
return abort(message, return_value=-1, fatal=fatal, logger=logger)
if overwrite is not None:
islink = os.path.islink(pdest)
if islink or os.path.exists(pdest):
if not overwrite:
message = "%s exists, can't %s" % (short(destination), action.lower())
return abort(message, return_value=-1, fatal=fatal, logger=logger)
_do_delete(pdest, islink, fatal)
try:
# Ensure parent folder exists
ensure_folder(parent_folder(destination), fatal=fatal, logger=None, dryrun=dryrun)
_R.hlog(logger, "%s%s" % (description[0].upper(), description[1:]))
if ignore is not None:
if not callable(ignore):
given = ignore
def ignore(*_):
return given
extra["ignore"] = ignore
func(source, destination, **extra)
return 1
except Exception as e:
return abort("Can't %s" % description, exc_info=e, return_value=-1, fatal=fatal, logger=logger)
def run(program,
*args,
background=False,
fatal=True,
logger=UNSET,
dryrun=UNSET,
short_exe=UNSET,
passthrough=False,
path_env=None,
strip="\r\n",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
**popen_args):
"""Run 'program' with 'args'
Args:
program (str | pathlib.Path): Program to run (full path, or basename)
*args: Command line args to call 'program' with
background (bool): When True, background the spawned process (detach from console and current process)
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
short_exe (str | bool | None): Try to log a compact representation of executable
passthrough (bool | file | None): If True-ish, pass-through stderr/stdout in addition to capturing it
as well as 'passthrough' itself if it has a write() function
path_env (dict | None): Allows to inject PATH-like env vars, see `_added_env_paths()`
strip (str | bool | None): If provided, `strip()` the captured output [default: strip "\n" newlines]
stdout (int | IO[Any] | None): Passed-through to subprocess.Popen, [default: subprocess.PIPE]
stderr (int | IO[Any] | None): Passed-through to subprocess.Popen, [default: subprocess.PIPE]
**popen_args: Passed through to `subprocess.Popen`
Returns:
(RunResult): Run outcome, use .failed, .succeeded, .output, .error etc to inspect the outcome
"""
if path_env:
popen_args["env"] = _added_env_paths(path_env,
env=popen_args.get("env"))
args = flattened(args, shellify=True)
full_path = which(program)
result = RunResult(audit=RunAudit(full_path or program, args, popen_args))
description = result.audit.run_description(short_exe=short_exe)
if background:
description += " &"
abort_logger = None if logger is None else UNSET
if logger is True or logger is print:
# When logger is True, we just print() the message, so we may as well color it nicely
description = _R.colored(description, "bold")
if _R.hdry(dryrun, logger, "run: %s" % description):
result.audit.dryrun = True
result.exit_code = 0
if stdout is not None:
result.output = "[dryrun] %s" % description # Properly simulate a successful run
if stdout is not None:
result.error = ""
return result
if not full_path:
if program and os.path.basename(program) == program:
result.error = "%s is not installed (PATH=%s)" % (
short(program), short(os.environ.get("PATH")))
else:
result.error = "%s is not an executable" % short(program)
return abort(result.error,
return_value=result,
fatal=fatal,
logger=abort_logger)
_R.hlog(logger, "Running: %s" % description)
if background:
child_pid = daemonize()
if child_pid:
result.pid = child_pid # In parent process, we just report a successful run (we don't wait/check on background process)
result.exit_code = 0
return result
fatal = False # pragma: no cover, non-fatal mode in background process (there is no more console etc to report anything)
with _WrappedArgs([full_path] + args) as wrapped_args:
try:
p, out, err = _run_popen(wrapped_args, popen_args, passthrough,
fatal, stdout, stderr)
result.output = decode(out, strip=strip)
result.error = decode(err, strip=strip)
result.pid = p.pid
result.exit_code = p.returncode
except Exception as e:
if fatal:
# Don't re-wrap with an abort(), let original stacktrace show through
raise
result.exc_info = e
if not result.error:
result.error = "%s failed: %s" % (
short(program), repr(e) if isinstance(e, OSError) else e)
if fatal and result.exit_code:
base_message = "%s exited with code %s" % (short(program),
result.exit_code)
if passthrough and (result.output or result.error):
exception = _R.abort_exception(override=fatal)
if exception is SystemExit:
raise SystemExit(result.exit_code)
if isinstance(exception, type) and issubclass(
exception, BaseException):
raise exception(base_message)
message = []
if abort_logger is not None and not passthrough:
# Log full output, unless user explicitly turned it off
message.append("Run failed: %s" % description)
if result.error:
message.append("\nstderr:\n%s" % result.error)
if result.output:
message.append("\nstdout:\n%s" % result.output)
message.append(base_message)
abort("\n".join(message),
code=result.exit_code,
exc_info=result.exc_info,
fatal=fatal,
logger=abort_logger)
if background:
os._exit(
result.exit_code
) # pragma: no cover, simply exit forked process (don't go back to caller)
return result