def read_json(path, default=None, fatal=True, logger=None):
"""
:param str|None path: Path to file to deserialize
:param dict|list|None default: Default if file is not present, or if it's not json
:param bool|None fatal: Abort execution on failure if True
:param callable|None logger: Logger to use
:return dict|list: Deserialized data from file
"""
path = resolved_path(path)
if not path or not os.path.exists(path):
if default is None:
return abort("No file %s", short(path), fatal=(fatal, default))
return default
try:
with io.open(path, "rt") as fh:
data = json.load(fh)
if default is not None and type(data) != type(default):
return abort("Wrong type %s for %s, expecting %s", type(data), short(path), type(default), fatal=(fatal, default))
if logger:
logger("Read %s", short(path))
return data
except Exception as e:
return abort("Couldn't read %s: %s", short(path), e, fatal=(fatal, default))
def make_executable(path, fatal=True, logger=UNSET, dryrun=UNSET):
"""
Args:
path (str): chmod file with 'path' as executable
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
Returns:
(int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed
"""
if is_executable(path):
return 0
if _R.hdry(dryrun, logger, "make %s executable" % short(path)):
return 1
if not os.path.exists(path):
return abort("%s does not exist, can't make it executable" %
short(path),
return_value=-1,
fatal=fatal,
logger=logger)
try:
os.chmod(path, 0o755) # nosec
_R.hlog(logger, "Made '%s' executable" % short(path))
return 1
except Exception as e:
return abort("Can't chmod %s" % short(path),
exc_info=e,
return_value=-1,
fatal=fatal,
logger=logger)
def readlines(path, first=None, errors="ignore", fatal=False, logger=False):
"""
Args:
path (str | Path | None): Path to file to read lines from
first (int | None): Return only the 'first' lines when specified
errors (str | None): Optional string specifying how encoding errors are to be handled
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
Yields:
(str): Lines read, newlines and trailing spaces stripped
"""
try:
with io.open(resolved_path(path), errors=errors) as fh:
if not first:
first = -1
for line in fh:
if first == 0:
return
yield decode(line).rstrip()
first -= 1
except Exception as e:
message = "Can't read %s" % short(path)
if fatal:
abort(_R.actual_message(message), exc_info=e, fatal=fatal, logger=logger)
_R.hlog(logger, message, exc_info=e)
def require_installed(program, instructions=None, platform=None):
"""Raise an exception if 'program' is not available on PATH, show instructions on how to install it
Args:
program (str): Program to check
instructions (str | dict): Short instructions letting user know how to get `program` installed, example: `run: brew install foo`
Extra convenience, specify:
- None if `program` can simply be installed via `brew install <program>`
- A word (without spaces) to refer to "usual" package (brew on OSX, apt on Linux etc.)
- A dict with per-platform instructions
platform (str | None): Override SYS_INFO.platform (for testing instructions rendering)
Returns:
(bool): True if installed, False otherwise (when fatal=False)
"""
if which(program) is None:
if not instructions:
instructions = dict(macos="run: `brew install {program}`",
linux="run: `apt install {program}`")
if isinstance(instructions, dict):
instructions = _install_instructions(
instructions, platform or SYS_INFO.platform_id.platform)
message = "{program} is not installed"
if instructions:
if "\n" in instructions:
message += ":\n- %s" % instructions
else:
message += ", %s" % instructions
message = message.format(program=program)
abort(message)
def _checked_pyinstall(self, python, fatal, logger):
"""Optionally abort if 'python' installation is not valid"""
if python.problem and fatal:
abort("Invalid python installation: %s" % python,
fatal=fatal,
logger=logger)
return python
def run(program, *args, **kwargs):
"""Run 'program' with 'args'"""
args = flattened(args, split=SHELL)
full_path = which(program)
logger = kwargs.pop("logger", LOG.debug)
fatal = kwargs.pop("fatal", True)
dryrun = kwargs.pop("dryrun", is_dryrun())
include_error = kwargs.pop("include_error", False)
message = "Would run" if dryrun else "Running"
message = "%s: %s %s" % (message, short(
full_path or program), represented_args(args))
if logger:
logger(message)
if dryrun:
return message
if not full_path:
return abort("%s is not installed", short(program), fatal=fatal)
stdout = kwargs.pop("stdout", subprocess.PIPE)
stderr = kwargs.pop("stderr", subprocess.PIPE)
args = [full_path] + args
try:
path_env = kwargs.pop("path_env", None)
if path_env:
kwargs["env"] = added_env_paths(path_env, env=kwargs.get("env"))
p = subprocess.Popen(args, stdout=stdout, stderr=stderr,
**kwargs) # nosec
output, err = p.communicate()
output = decode(output, strip=True)
err = decode(err, strip=True)
if p.returncode and fatal is not None:
note = ": %s\n%s" % (err, output) if output or err else ""
message = "%s exited with code %s%s" % (short(program),
p.returncode, note.strip())
return abort(message, fatal=fatal)
if include_error and err:
output = "%s\n%s" % (output, err)
return output and output.strip()
except Exception as e:
return abort("%s failed: %s",
short(program),
e,
exc_info=e,
fatal=fatal)
def ensure_folder(path,
folder=False,
fatal=True,
logger=LOG.debug,
dryrun=None):
"""
:param str|None path: Path to file or folder
:param bool folder: If True, 'path' refers to a folder (file otherwise)
:param bool|None fatal: Abort execution on failure if True
:param callable|None logger: Logger to use
:param bool|None dryrun: If specified, override global is_dryrun()
:return int: 1 if effectively done, 0 if no-op, -1 on failure
"""
if not path:
return 0
if folder:
folder = resolved_path(path)
else:
folder = parent_folder(path)
if os.path.isdir(folder):
if not os.access(folder, os.W_OK):
return abort("Folder %s is not writable",
folder,
fatal=(fatal, -1),
logger=logger)
return 0
if dryrun is None:
dryrun = is_dryrun()
if dryrun:
LOG.debug("Would create %s", short(folder))
return 1
try:
os.makedirs(folder)
if logger:
logger("Created folder %s", short(folder))
return 1
except Exception as e:
return abort("Can't create folder %s: %s",
short(folder),
e,
fatal=(fatal, -1),
logger=logger)
def delete(path, fatal=True, logger=LOG.debug):
"""
:param str|None path: Path to file or folder to delete
:param bool|None fatal: Abort execution on failure if True
:param callable|None logger: Logger to use
:return int: 1 if effectively done, 0 if no-op, -1 on failure
"""
islink = path and os.path.islink(path)
if not islink and (not path or not os.path.exists(path)):
return 0
if is_dryrun():
LOG.debug("Would delete %s", short(path))
return 1
if logger:
logger("Deleting %s", short(path))
try:
if islink or os.path.isfile(path):
os.unlink(path)
else:
shutil.rmtree(path)
return 1
except Exception as e:
return abort("Can't delete %s: %s", short(path), e, fatal=(fatal, -1))
def delete(path, fatal=True, logger=UNSET, dryrun=UNSET):
"""
Args:
path (str | Path | None): Path to file or folder to delete
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
Returns:
(int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed
"""
path = resolved_path(path)
islink = path and os.path.islink(path)
if not islink and (not path or not os.path.exists(path)):
return 0
if _R.hdry(dryrun, logger, "delete %s" % short(path)):
return 1
try:
_do_delete(path, islink, fatal)
_R.hlog(logger, "Deleted %s" % short(path))
return 1
except Exception as e:
return abort("Can't delete %s" % short(path), exc_info=e, return_value=-1, fatal=fatal, logger=logger)
def write(path, contents, fatal=True, logger=None):
"""
:param str|None path: Path to file
:param str|None contents: Contents to write
:param bool|None fatal: Abort execution on failure if True
:param callable|None logger: Logger to use
:return int: 1 if effectively done, 0 if no-op, -1 on failure
"""
if not path:
return 0
if is_dryrun():
action = "write %s bytes to" % len(contents) if contents else "touch"
LOG.debug("Would %s %s", action, short(path))
return 1
ensure_folder(path, fatal=fatal, logger=logger)
if logger and contents:
logger("Writing %s bytes to %s", len(contents), short(path))
try:
with io.open(path, "wt") as fh:
if contents:
fh.write(decode(contents))
else:
os.utime(path, None)
return 1
except Exception as e:
return abort("Can't write to %s: %s",
short(path),
e,
fatal=(fatal, -1))
def download(self, url, destination, fatal=True, logger=UNSET, dryrun=UNSET, **kwargs):
"""
Args:
url (str): URL of resource to download (may be absolute, or relative to self.base_url)
Use #sha256=... or #sha512=... at the end of the url to ensure content is validated against given checksum
destination (str | Path): Path to local file where to store the download
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
**kwargs: Passed through to underlying client
Returns:
(RestResponse): Response from underlying call
"""
hash_algo, hash_checksum, url = self._decomposed_checksum_url(url)
response = self._get_response("GET", url, fatal, logger, dryrun=dryrun, action="download", **kwargs)
if response.ok and not _R.resolved_dryrun(dryrun):
destination = to_path(destination)
ensure_folder(destination.parent, fatal=fatal, logger=None)
with open(destination, "wb") as fh:
fh.write(response.content)
if hash_checksum:
downloaded_checksum = checksum(destination, hash=hash_algo)
if downloaded_checksum != hash_checksum:
delete(destination, fatal=False, logger=logger)
msg = "%s differs for %s: expecting %s, got %s" % (hash_algo, short(destination), hash_checksum, downloaded_checksum)
return abort(msg, fatal=fatal, logger=logger)
return response
def decompress(source, destination, ext=None, overwrite=True, simplify=False, fatal=True, logger=UNSET, dryrun=UNSET):
"""
Args:
source (str | Path | None): Source file to decompress
destination (str | Path | None): Destination folder
ext (str | None): Extension determining compression (default: extension of given 'source' file)
overwrite (bool | None): True: replace existing, False: fail if destination exists, None: no destination check
simplify (bool): If True and source has only one sub-folder, extract that one sub-folder to destination
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
Returns:
(int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed
"""
if not ext:
_, _, ext = str(source).lower().rpartition(".")
ext = SYS_INFO.platform_id.canonical_compress_extension(ext, short_form=True)
if not ext:
message = "Unknown extension '%s': can't decompress file" % os.path.basename(source)
return abort(message, return_value=-1, fatal=fatal, logger=logger)
func = _unzip if ext == "zip" else _untar
return _file_op(source, destination, func, overwrite, fatal, logger, dryrun, simplify=simplify)
def _get_response(self, method, url, fatal, logger, dryrun=False, state=None, action=None, **kwargs):
"""
Args:
method (str): Underlying method to call
url (str): Remote URL (may be absolute, or relative to self.base_url)
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
state (DataState | None): For PUT/POST requests
action (str | None): Action to refer to in dryrun message (default: method)
**kwargs: Passed through to underlying client
Returns:
(RestResponse): Response from underlying call
"""
absolute_url = self.full_url(url)
message = "%s %s" % (action or method, absolute_url)
if _R.hdry(dryrun, logger, message):
return RestResponse(method, absolute_url, MockResponse(200, dict(message="dryrun %s" % message)))
full_headers = self.headers
headers = kwargs.get("headers")
if headers:
full_headers = dict(full_headers)
full_headers.update(headers)
keyword_args = dict(kwargs)
keyword_args["headers"] = full_headers
keyword_args.setdefault("timeout", self.timeout)
if state is not None:
state.complete(keyword_args)
try:
raw_response = self._protected_get(method, absolute_url, keyword_args)
response = self.handler.to_rest_response(method, absolute_url, raw_response)
if fatal or logger is not None:
msg = response.description()
if fatal and not response.ok:
abort(msg, fatal=fatal, logger=logger)
_R.hlog(logger, msg)
return response
finally:
if state is not None:
state.close()
def to_path(path, no_spaces=False):
"""
Args:
path (str | Path): Path to convert
no_spaces (type | bool | None): If True-ish, abort if 'path' contains a space
Returns:
(Path | None): Converted to `Path` object, if necessary
"""
if no_spaces and " " in str(path):
abort("Refusing path with space (not worth escaping all the things to make this work): '%s'" % short(path), fatal=no_spaces)
if isinstance(path, Path):
return path
if path is not None:
if path:
path = os.path.expanduser(path)
return Path(path)
def save_json(data, path, fatal=True, logger=None, sort_keys=True, indent=2, **kwargs):
"""
Args:
data (object | None): Data to serialize and save
path (str | None): Path to file where to save
fatal (bool | None): Abort execution on failure if True
logger (callable | None): Logger to use
sort_keys (bool): Save json with sorted keys
indent (int): Indentation to use
**kwargs: Passed through to `json.dump()`
Returns:
(int): 1 if saved, -1 if failed (when `fatal` is False)
"""
if data is None or not path:
return abort("No file %s", short(path), fatal=fatal)
try:
path = resolved_path(path)
ensure_folder(path, fatal=fatal, logger=None)
if is_dryrun():
LOG.info("Would save %s", short(path))
return 1
if hasattr(data, "to_dict"):
data = data.to_dict()
if indent:
kwargs.setdefault("separators", (",", ': '))
with open(path, "wt") as fh:
json.dump(data, fh, sort_keys=sort_keys, indent=indent, **kwargs)
fh.write("\n")
if logger:
logger("Saved %s", short(path))
return 1
except Exception as e:
return abort("Couldn't save %s: %s", short(path), e, fatal=(fatal, -1))
def make_executable(path, fatal=True):
"""
:param str|None path: chmod file with 'path' as executable
:param bool|None fatal: Abort execution on failure if True
:return int: 1 if effectively done, 0 if no-op, -1 on failure
"""
if is_executable(path):
return 0
if is_dryrun():
LOG.debug("Would make %s executable", short(path))
return 1
if not os.path.exists(path):
return abort("%s does not exist, can't make it executable",
short(path),
fatal=(fatal, -1))
try:
os.chmod(path, 0o755) # nosec
return 1
except Exception as e:
return abort("Can't chmod %s: %s", short(path), e, fatal=(fatal, -1))
def ensure_folder(path, clean=False, fatal=True, logger=UNSET, dryrun=UNSET):
"""Ensure folder with 'path' exists
Args:
path (str | Path | None): Path to file or folder
clean (bool): True: If True, ensure folder is clean (delete any file/folder it may have)
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
Returns:
(int): In non-fatal mode, >=1: successfully done, 0: was no-op, -1: failed
"""
path = resolved_path(path)
if not path:
return 0
if os.path.isdir(path):
if not clean:
return 0
cleaned = 0
for fname in os.listdir(path):
cleaned += delete(os.path.join(path, fname), fatal=fatal, logger=None, dryrun=dryrun)
if cleaned:
msg = "%s from %s" % (_R.lc.rm.plural(cleaned, "file"), short(path))
if not _R.hdry(dryrun, logger, "clean %s" % msg):
_R.hlog(logger, "Cleaned %s" % msg)
return cleaned
if _R.hdry(dryrun, logger, "create %s" % short(path)):
return 1
try:
os.makedirs(path)
_R.hlog(logger, "Created folder %s" % short(path))
return 1
except Exception as e:
return abort("Can't create folder %s" % short(path), exc_info=e, return_value=-1, fatal=fatal, logger=logger)
def write(path, contents, fatal=True, logger=UNSET, dryrun=UNSET):
"""Write `contents` to file with `path`
Args:
path (str | Path | None): Path to file
contents (str | bytes | None): Contents to write (only touch file if None)
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
Returns:
(int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed
"""
if not path:
return 0
path = resolved_path(path)
byte_size = _R.lc.rm.represented_bytesize(len(contents), unit="bytes") if contents else ""
def dryrun_msg():
return "%s %s" % ("write %s to" % byte_size if byte_size else "touch", short(path))
if _R.hdry(dryrun, logger, dryrun_msg):
return 1
ensure_folder(parent_folder(path), fatal=fatal, logger=None, dryrun=dryrun)
try:
mode = "wb" if isinstance(contents, bytes) else "wt"
with io.open(path, mode) as fh:
if contents is None:
os.utime(path, None)
else:
fh.write(contents)
_R.hlog(logger, "%s %s" % ("Wrote %s to" % byte_size if byte_size else "Touched", short(path)))
return 1
except Exception as e:
return abort("Can't write to %s" % short(path), exc_info=e, return_value=-1, fatal=fatal, logger=logger)
def get_lines(path, max_size=TEXT_THRESHOLD_SIZE, fatal=True, default=None):
"""
:param str|None path: Path of text file to return lines from
:param int|None max_size: Return contents only for files smaller than 'max_size' bytes
:param bool|None fatal: Abort execution on failure if True
:param list|None default: Object to return if lines couldn't be read
:return list|None: Lines from file contents
"""
if not path or not os.path.isfile(path) or (
max_size and os.path.getsize(path) > max_size):
# Intended for small text files, pretend no contents for binaries
return default
try:
with io.open(path, "rt", errors="ignore") as fh:
return fh.readlines()
except Exception as e:
return abort("Can't read %s: %s",
short(path),
e,
fatal=(fatal, default))
def compress(source, destination, arcname=UNSET, ext=None, overwrite=True, fatal=True, logger=UNSET, dryrun=UNSET):
"""
Args:
source (str | Path | None): Source folder to compress
destination (str | Path | None): Destination folder
arcname (str | None): Name of subfolder in archive (default: source basename)
ext (str | None): Extension determining compression (default: extension of given 'source' file)
overwrite (bool | None): True: replace existing, False: fail if destination exists, None: no destination check
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
Returns:
(int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed
"""
if not ext:
_, _, ext = str(destination).lower().rpartition(".")
kwargs = {}
ext = SYS_INFO.platform_id.canonical_compress_extension(ext, short_form=True)
if not ext:
message = "Unknown extension '%s': can't compress file" % os.path.basename(destination)
return abort(message, return_value=-1, fatal=fatal, logger=logger)
if arcname is UNSET:
arcname = os.path.basename(source)
arcname = to_path(arcname or "")
if ext == "zip":
func = _zip
else:
func = _tar
kwargs["mode"] = "w:" if ext == "tar" else "w:%s" % ext
return _file_op(source, destination, func, overwrite, fatal, logger, dryrun, arcname=arcname, **kwargs)
def _file_op(source,
destination,
func,
adapter,
fatal,
logger,
must_exist=True):
"""
Call func(source, destination)
:param str|None source: Source file or folder
:param str|None destination: Destination file or folder
:param callable func: Implementation function
:param callable adapter: Optional function to call on 'source' before copy
:param bool|None fatal: Abort execution on failure if True
:param callable|None logger: Logger to use
:param bool must_exist: If True, verify that source does indeed exist
:return int: 1 if effectively done, 0 if no-op, -1 on failure
"""
if not source or not destination or source == destination:
return 0
action = func.__name__[1:]
indicator = "<-" if action == "symlink" else "->"
psource = parent_folder(source)
pdest = resolved_path(destination)
if psource != pdest and psource.startswith(pdest):
return abort("Can't %s %s %s %s: source contained in destination",
action,
short(source),
indicator,
short(destination),
fatal=(fatal, -1))
if is_dryrun():
LOG.debug("Would %s %s %s %s", action, short(source), indicator,
short(destination))
return 1
if must_exist and not os.path.exists(source):
return abort("%s does not exist, can't %s to %s",
short(source),
action.title(),
short(destination),
fatal=(fatal, -1))
try:
# Delete destination, but ensure that its parent folder exists
delete(destination, fatal=fatal, logger=None)
ensure_folder(destination, fatal=fatal, logger=None)
if logger:
note = adapter(source, destination, fatal=fatal,
logger=logger) if adapter else ""
if logger:
logger("%s %s %s %s%s", action.title(), short(source),
indicator, short(destination), note)
func(source, destination)
return 1
except Exception as e:
return abort("Can't %s %s %s %s: %s",
action,
short(source),
indicator,
short(destination),
e,
fatal=(fatal, -1))
def run(program,
*args,
background=False,
fatal=True,
logger=UNSET,
dryrun=UNSET,
short_exe=UNSET,
passthrough=False,
path_env=None,
strip="\r\n",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
**popen_args):
"""Run 'program' with 'args'
Args:
program (str | pathlib.Path): Program to run (full path, or basename)
*args: Command line args to call 'program' with
background (bool): When True, background the spawned process (detach from console and current process)
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
short_exe (str | bool | None): Try to log a compact representation of executable
passthrough (bool | file | None): If True-ish, pass-through stderr/stdout in addition to capturing it
as well as 'passthrough' itself if it has a write() function
path_env (dict | None): Allows to inject PATH-like env vars, see `_added_env_paths()`
strip (str | bool | None): If provided, `strip()` the captured output [default: strip "\n" newlines]
stdout (int | IO[Any] | None): Passed-through to subprocess.Popen, [default: subprocess.PIPE]
stderr (int | IO[Any] | None): Passed-through to subprocess.Popen, [default: subprocess.PIPE]
**popen_args: Passed through to `subprocess.Popen`
Returns:
(RunResult): Run outcome, use .failed, .succeeded, .output, .error etc to inspect the outcome
"""
if path_env:
popen_args["env"] = _added_env_paths(path_env,
env=popen_args.get("env"))
args = flattened(args, shellify=True)
full_path = which(program)
result = RunResult(audit=RunAudit(full_path or program, args, popen_args))
description = result.audit.run_description(short_exe=short_exe)
if background:
description += " &"
abort_logger = None if logger is None else UNSET
if logger is True or logger is print:
# When logger is True, we just print() the message, so we may as well color it nicely
description = _R.colored(description, "bold")
if _R.hdry(dryrun, logger, "run: %s" % description):
result.audit.dryrun = True
result.exit_code = 0
if stdout is not None:
result.output = "[dryrun] %s" % description # Properly simulate a successful run
if stdout is not None:
result.error = ""
return result
if not full_path:
if program and os.path.basename(program) == program:
result.error = "%s is not installed (PATH=%s)" % (
short(program), short(os.environ.get("PATH")))
else:
result.error = "%s is not an executable" % short(program)
return abort(result.error,
return_value=result,
fatal=fatal,
logger=abort_logger)
_R.hlog(logger, "Running: %s" % description)
if background:
child_pid = daemonize()
if child_pid:
result.pid = child_pid # In parent process, we just report a successful run (we don't wait/check on background process)
result.exit_code = 0
return result
fatal = False # pragma: no cover, non-fatal mode in background process (there is no more console etc to report anything)
with _WrappedArgs([full_path] + args) as wrapped_args:
try:
p, out, err = _run_popen(wrapped_args, popen_args, passthrough,
fatal, stdout, stderr)
result.output = decode(out, strip=strip)
result.error = decode(err, strip=strip)
result.pid = p.pid
result.exit_code = p.returncode
except Exception as e:
if fatal:
# Don't re-wrap with an abort(), let original stacktrace show through
raise
result.exc_info = e
if not result.error:
result.error = "%s failed: %s" % (
short(program), repr(e) if isinstance(e, OSError) else e)
if fatal and result.exit_code:
base_message = "%s exited with code %s" % (short(program),
result.exit_code)
if passthrough and (result.output or result.error):
exception = _R.abort_exception(override=fatal)
if exception is SystemExit:
raise SystemExit(result.exit_code)
if isinstance(exception, type) and issubclass(
exception, BaseException):
raise exception(base_message)
message = []
if abort_logger is not None and not passthrough:
# Log full output, unless user explicitly turned it off
message.append("Run failed: %s" % description)
if result.error:
message.append("\nstderr:\n%s" % result.error)
if result.output:
message.append("\nstdout:\n%s" % result.output)
message.append(base_message)
abort("\n".join(message),
code=result.exit_code,
exc_info=result.exc_info,
fatal=fatal,
logger=abort_logger)
if background:
os._exit(
result.exit_code
) # pragma: no cover, simply exit forked process (don't go back to caller)
return result
def _file_op(source, destination, func, overwrite, fatal, logger, dryrun, must_exist=True, ignore=None, **extra):
"""Call func(source, destination)
Args:
source (str | None): Source file or folder
destination (str | None): Destination file or folder
func (callable): Implementation function
overwrite (bool | None): True: replace existing, False: fail if destination exists, None: no destination check
fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log
logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter
dryrun (bool | UNSET | None): Optionally override current dryrun setting
must_exist (bool): If True, verify that source does indeed exist
ignore (callable | list | str | None): Names to be ignored
**extra: Passed-through to 'func'
Returns:
(int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed
"""
if not source or not destination or source == destination:
return 0
action = func.__name__[1:]
indicator = "<-" if action == "symlink" else "->"
description = "%s %s %s %s" % (action, short(source), indicator, short(destination))
psource = parent_folder(source)
pdest = resolved_path(destination)
if psource.startswith(pdest):
message = "Can't %s: source contained in destination" % description
return abort(message, return_value=-1, fatal=fatal, logger=logger)
if _R.hdry(dryrun, logger, description):
return 1
if must_exist and not (os.path.exists(source) or os.path.islink(source)):
message = "%s does not exist, can't %s to %s" % (short(source), action.lower(), short(destination))
return abort(message, return_value=-1, fatal=fatal, logger=logger)
if overwrite is not None:
islink = os.path.islink(pdest)
if islink or os.path.exists(pdest):
if not overwrite:
message = "%s exists, can't %s" % (short(destination), action.lower())
return abort(message, return_value=-1, fatal=fatal, logger=logger)
_do_delete(pdest, islink, fatal)
try:
# Ensure parent folder exists
ensure_folder(parent_folder(destination), fatal=fatal, logger=None, dryrun=dryrun)
_R.hlog(logger, "%s%s" % (description[0].upper(), description[1:]))
if ignore is not None:
if not callable(ignore):
given = ignore
def ignore(*_):
return given
extra["ignore"] = ignore
func(source, destination, **extra)
return 1
except Exception as e:
return abort("Can't %s" % description, exc_info=e, return_value=-1, fatal=fatal, logger=logger)
