def read_json(path, default=None, fatal=True, logger=None): """ :param str|None path: Path to file to deserialize :param dict|list|None default: Default if file is not present, or if it's not json :param bool|None fatal: Abort execution on failure if True :param callable|None logger: Logger to use :return dict|list: Deserialized data from file """ path = resolved_path(path) if not path or not os.path.exists(path): if default is None: return abort("No file %s", short(path), fatal=(fatal, default)) return default try: with io.open(path, "rt") as fh: data = json.load(fh) if default is not None and type(data) != type(default): return abort("Wrong type %s for %s, expecting %s", type(data), short(path), type(default), fatal=(fatal, default)) if logger: logger("Read %s", short(path)) return data except Exception as e: return abort("Couldn't read %s: %s", short(path), e, fatal=(fatal, default))
def make_executable(path, fatal=True, logger=UNSET, dryrun=UNSET): """ Args: path (str): chmod file with 'path' as executable fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter dryrun (bool | UNSET | None): Optionally override current dryrun setting Returns: (int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed """ if is_executable(path): return 0 if _R.hdry(dryrun, logger, "make %s executable" % short(path)): return 1 if not os.path.exists(path): return abort("%s does not exist, can't make it executable" % short(path), return_value=-1, fatal=fatal, logger=logger) try: os.chmod(path, 0o755) # nosec _R.hlog(logger, "Made '%s' executable" % short(path)) return 1 except Exception as e: return abort("Can't chmod %s" % short(path), exc_info=e, return_value=-1, fatal=fatal, logger=logger)
def readlines(path, first=None, errors="ignore", fatal=False, logger=False): """ Args: path (str | Path | None): Path to file to read lines from first (int | None): Return only the 'first' lines when specified errors (str | None): Optional string specifying how encoding errors are to be handled fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter Yields: (str): Lines read, newlines and trailing spaces stripped """ try: with io.open(resolved_path(path), errors=errors) as fh: if not first: first = -1 for line in fh: if first == 0: return yield decode(line).rstrip() first -= 1 except Exception as e: message = "Can't read %s" % short(path) if fatal: abort(_R.actual_message(message), exc_info=e, fatal=fatal, logger=logger) _R.hlog(logger, message, exc_info=e)
def require_installed(program, instructions=None, platform=None): """Raise an exception if 'program' is not available on PATH, show instructions on how to install it Args: program (str): Program to check instructions (str | dict): Short instructions letting user know how to get `program` installed, example: `run: brew install foo` Extra convenience, specify: - None if `program` can simply be installed via `brew install <program>` - A word (without spaces) to refer to "usual" package (brew on OSX, apt on Linux etc.) - A dict with per-platform instructions platform (str | None): Override SYS_INFO.platform (for testing instructions rendering) Returns: (bool): True if installed, False otherwise (when fatal=False) """ if which(program) is None: if not instructions: instructions = dict(macos="run: `brew install {program}`", linux="run: `apt install {program}`") if isinstance(instructions, dict): instructions = _install_instructions( instructions, platform or SYS_INFO.platform_id.platform) message = "{program} is not installed" if instructions: if "\n" in instructions: message += ":\n- %s" % instructions else: message += ", %s" % instructions message = message.format(program=program) abort(message)
def _checked_pyinstall(self, python, fatal, logger): """Optionally abort if 'python' installation is not valid""" if python.problem and fatal: abort("Invalid python installation: %s" % python, fatal=fatal, logger=logger) return python
def run(program, *args, **kwargs): """Run 'program' with 'args'""" args = flattened(args, split=SHELL) full_path = which(program) logger = kwargs.pop("logger", LOG.debug) fatal = kwargs.pop("fatal", True) dryrun = kwargs.pop("dryrun", is_dryrun()) include_error = kwargs.pop("include_error", False) message = "Would run" if dryrun else "Running" message = "%s: %s %s" % (message, short( full_path or program), represented_args(args)) if logger: logger(message) if dryrun: return message if not full_path: return abort("%s is not installed", short(program), fatal=fatal) stdout = kwargs.pop("stdout", subprocess.PIPE) stderr = kwargs.pop("stderr", subprocess.PIPE) args = [full_path] + args try: path_env = kwargs.pop("path_env", None) if path_env: kwargs["env"] = added_env_paths(path_env, env=kwargs.get("env")) p = subprocess.Popen(args, stdout=stdout, stderr=stderr, **kwargs) # nosec output, err = p.communicate() output = decode(output, strip=True) err = decode(err, strip=True) if p.returncode and fatal is not None: note = ": %s\n%s" % (err, output) if output or err else "" message = "%s exited with code %s%s" % (short(program), p.returncode, note.strip()) return abort(message, fatal=fatal) if include_error and err: output = "%s\n%s" % (output, err) return output and output.strip() except Exception as e: return abort("%s failed: %s", short(program), e, exc_info=e, fatal=fatal)
def ensure_folder(path, folder=False, fatal=True, logger=LOG.debug, dryrun=None): """ :param str|None path: Path to file or folder :param bool folder: If True, 'path' refers to a folder (file otherwise) :param bool|None fatal: Abort execution on failure if True :param callable|None logger: Logger to use :param bool|None dryrun: If specified, override global is_dryrun() :return int: 1 if effectively done, 0 if no-op, -1 on failure """ if not path: return 0 if folder: folder = resolved_path(path) else: folder = parent_folder(path) if os.path.isdir(folder): if not os.access(folder, os.W_OK): return abort("Folder %s is not writable", folder, fatal=(fatal, -1), logger=logger) return 0 if dryrun is None: dryrun = is_dryrun() if dryrun: LOG.debug("Would create %s", short(folder)) return 1 try: os.makedirs(folder) if logger: logger("Created folder %s", short(folder)) return 1 except Exception as e: return abort("Can't create folder %s: %s", short(folder), e, fatal=(fatal, -1), logger=logger)
def delete(path, fatal=True, logger=LOG.debug): """ :param str|None path: Path to file or folder to delete :param bool|None fatal: Abort execution on failure if True :param callable|None logger: Logger to use :return int: 1 if effectively done, 0 if no-op, -1 on failure """ islink = path and os.path.islink(path) if not islink and (not path or not os.path.exists(path)): return 0 if is_dryrun(): LOG.debug("Would delete %s", short(path)) return 1 if logger: logger("Deleting %s", short(path)) try: if islink or os.path.isfile(path): os.unlink(path) else: shutil.rmtree(path) return 1 except Exception as e: return abort("Can't delete %s: %s", short(path), e, fatal=(fatal, -1))
def delete(path, fatal=True, logger=UNSET, dryrun=UNSET): """ Args: path (str | Path | None): Path to file or folder to delete fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter dryrun (bool | UNSET | None): Optionally override current dryrun setting Returns: (int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed """ path = resolved_path(path) islink = path and os.path.islink(path) if not islink and (not path or not os.path.exists(path)): return 0 if _R.hdry(dryrun, logger, "delete %s" % short(path)): return 1 try: _do_delete(path, islink, fatal) _R.hlog(logger, "Deleted %s" % short(path)) return 1 except Exception as e: return abort("Can't delete %s" % short(path), exc_info=e, return_value=-1, fatal=fatal, logger=logger)
def write(path, contents, fatal=True, logger=None): """ :param str|None path: Path to file :param str|None contents: Contents to write :param bool|None fatal: Abort execution on failure if True :param callable|None logger: Logger to use :return int: 1 if effectively done, 0 if no-op, -1 on failure """ if not path: return 0 if is_dryrun(): action = "write %s bytes to" % len(contents) if contents else "touch" LOG.debug("Would %s %s", action, short(path)) return 1 ensure_folder(path, fatal=fatal, logger=logger) if logger and contents: logger("Writing %s bytes to %s", len(contents), short(path)) try: with io.open(path, "wt") as fh: if contents: fh.write(decode(contents)) else: os.utime(path, None) return 1 except Exception as e: return abort("Can't write to %s: %s", short(path), e, fatal=(fatal, -1))
def download(self, url, destination, fatal=True, logger=UNSET, dryrun=UNSET, **kwargs): """ Args: url (str): URL of resource to download (may be absolute, or relative to self.base_url) Use #sha256=... or #sha512=... at the end of the url to ensure content is validated against given checksum destination (str | Path): Path to local file where to store the download fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter dryrun (bool | UNSET | None): Optionally override current dryrun setting **kwargs: Passed through to underlying client Returns: (RestResponse): Response from underlying call """ hash_algo, hash_checksum, url = self._decomposed_checksum_url(url) response = self._get_response("GET", url, fatal, logger, dryrun=dryrun, action="download", **kwargs) if response.ok and not _R.resolved_dryrun(dryrun): destination = to_path(destination) ensure_folder(destination.parent, fatal=fatal, logger=None) with open(destination, "wb") as fh: fh.write(response.content) if hash_checksum: downloaded_checksum = checksum(destination, hash=hash_algo) if downloaded_checksum != hash_checksum: delete(destination, fatal=False, logger=logger) msg = "%s differs for %s: expecting %s, got %s" % (hash_algo, short(destination), hash_checksum, downloaded_checksum) return abort(msg, fatal=fatal, logger=logger) return response
def decompress(source, destination, ext=None, overwrite=True, simplify=False, fatal=True, logger=UNSET, dryrun=UNSET): """ Args: source (str | Path | None): Source file to decompress destination (str | Path | None): Destination folder ext (str | None): Extension determining compression (default: extension of given 'source' file) overwrite (bool | None): True: replace existing, False: fail if destination exists, None: no destination check simplify (bool): If True and source has only one sub-folder, extract that one sub-folder to destination fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter dryrun (bool | UNSET | None): Optionally override current dryrun setting Returns: (int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed """ if not ext: _, _, ext = str(source).lower().rpartition(".") ext = SYS_INFO.platform_id.canonical_compress_extension(ext, short_form=True) if not ext: message = "Unknown extension '%s': can't decompress file" % os.path.basename(source) return abort(message, return_value=-1, fatal=fatal, logger=logger) func = _unzip if ext == "zip" else _untar return _file_op(source, destination, func, overwrite, fatal, logger, dryrun, simplify=simplify)
def _get_response(self, method, url, fatal, logger, dryrun=False, state=None, action=None, **kwargs): """ Args: method (str): Underlying method to call url (str): Remote URL (may be absolute, or relative to self.base_url) fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter dryrun (bool | UNSET | None): Optionally override current dryrun setting state (DataState | None): For PUT/POST requests action (str | None): Action to refer to in dryrun message (default: method) **kwargs: Passed through to underlying client Returns: (RestResponse): Response from underlying call """ absolute_url = self.full_url(url) message = "%s %s" % (action or method, absolute_url) if _R.hdry(dryrun, logger, message): return RestResponse(method, absolute_url, MockResponse(200, dict(message="dryrun %s" % message))) full_headers = self.headers headers = kwargs.get("headers") if headers: full_headers = dict(full_headers) full_headers.update(headers) keyword_args = dict(kwargs) keyword_args["headers"] = full_headers keyword_args.setdefault("timeout", self.timeout) if state is not None: state.complete(keyword_args) try: raw_response = self._protected_get(method, absolute_url, keyword_args) response = self.handler.to_rest_response(method, absolute_url, raw_response) if fatal or logger is not None: msg = response.description() if fatal and not response.ok: abort(msg, fatal=fatal, logger=logger) _R.hlog(logger, msg) return response finally: if state is not None: state.close()
def to_path(path, no_spaces=False): """ Args: path (str | Path): Path to convert no_spaces (type | bool | None): If True-ish, abort if 'path' contains a space Returns: (Path | None): Converted to `Path` object, if necessary """ if no_spaces and " " in str(path): abort("Refusing path with space (not worth escaping all the things to make this work): '%s'" % short(path), fatal=no_spaces) if isinstance(path, Path): return path if path is not None: if path: path = os.path.expanduser(path) return Path(path)
def save_json(data, path, fatal=True, logger=None, sort_keys=True, indent=2, **kwargs): """ Args: data (object | None): Data to serialize and save path (str | None): Path to file where to save fatal (bool | None): Abort execution on failure if True logger (callable | None): Logger to use sort_keys (bool): Save json with sorted keys indent (int): Indentation to use **kwargs: Passed through to `json.dump()` Returns: (int): 1 if saved, -1 if failed (when `fatal` is False) """ if data is None or not path: return abort("No file %s", short(path), fatal=fatal) try: path = resolved_path(path) ensure_folder(path, fatal=fatal, logger=None) if is_dryrun(): LOG.info("Would save %s", short(path)) return 1 if hasattr(data, "to_dict"): data = data.to_dict() if indent: kwargs.setdefault("separators", (",", ': ')) with open(path, "wt") as fh: json.dump(data, fh, sort_keys=sort_keys, indent=indent, **kwargs) fh.write("\n") if logger: logger("Saved %s", short(path)) return 1 except Exception as e: return abort("Couldn't save %s: %s", short(path), e, fatal=(fatal, -1))
def make_executable(path, fatal=True): """ :param str|None path: chmod file with 'path' as executable :param bool|None fatal: Abort execution on failure if True :return int: 1 if effectively done, 0 if no-op, -1 on failure """ if is_executable(path): return 0 if is_dryrun(): LOG.debug("Would make %s executable", short(path)) return 1 if not os.path.exists(path): return abort("%s does not exist, can't make it executable", short(path), fatal=(fatal, -1)) try: os.chmod(path, 0o755) # nosec return 1 except Exception as e: return abort("Can't chmod %s: %s", short(path), e, fatal=(fatal, -1))
def ensure_folder(path, clean=False, fatal=True, logger=UNSET, dryrun=UNSET): """Ensure folder with 'path' exists Args: path (str | Path | None): Path to file or folder clean (bool): True: If True, ensure folder is clean (delete any file/folder it may have) fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter dryrun (bool | UNSET | None): Optionally override current dryrun setting Returns: (int): In non-fatal mode, >=1: successfully done, 0: was no-op, -1: failed """ path = resolved_path(path) if not path: return 0 if os.path.isdir(path): if not clean: return 0 cleaned = 0 for fname in os.listdir(path): cleaned += delete(os.path.join(path, fname), fatal=fatal, logger=None, dryrun=dryrun) if cleaned: msg = "%s from %s" % (_R.lc.rm.plural(cleaned, "file"), short(path)) if not _R.hdry(dryrun, logger, "clean %s" % msg): _R.hlog(logger, "Cleaned %s" % msg) return cleaned if _R.hdry(dryrun, logger, "create %s" % short(path)): return 1 try: os.makedirs(path) _R.hlog(logger, "Created folder %s" % short(path)) return 1 except Exception as e: return abort("Can't create folder %s" % short(path), exc_info=e, return_value=-1, fatal=fatal, logger=logger)
def write(path, contents, fatal=True, logger=UNSET, dryrun=UNSET): """Write `contents` to file with `path` Args: path (str | Path | None): Path to file contents (str | bytes | None): Contents to write (only touch file if None) fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter dryrun (bool | UNSET | None): Optionally override current dryrun setting Returns: (int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed """ if not path: return 0 path = resolved_path(path) byte_size = _R.lc.rm.represented_bytesize(len(contents), unit="bytes") if contents else "" def dryrun_msg(): return "%s %s" % ("write %s to" % byte_size if byte_size else "touch", short(path)) if _R.hdry(dryrun, logger, dryrun_msg): return 1 ensure_folder(parent_folder(path), fatal=fatal, logger=None, dryrun=dryrun) try: mode = "wb" if isinstance(contents, bytes) else "wt" with io.open(path, mode) as fh: if contents is None: os.utime(path, None) else: fh.write(contents) _R.hlog(logger, "%s %s" % ("Wrote %s to" % byte_size if byte_size else "Touched", short(path))) return 1 except Exception as e: return abort("Can't write to %s" % short(path), exc_info=e, return_value=-1, fatal=fatal, logger=logger)
def get_lines(path, max_size=TEXT_THRESHOLD_SIZE, fatal=True, default=None): """ :param str|None path: Path of text file to return lines from :param int|None max_size: Return contents only for files smaller than 'max_size' bytes :param bool|None fatal: Abort execution on failure if True :param list|None default: Object to return if lines couldn't be read :return list|None: Lines from file contents """ if not path or not os.path.isfile(path) or ( max_size and os.path.getsize(path) > max_size): # Intended for small text files, pretend no contents for binaries return default try: with io.open(path, "rt", errors="ignore") as fh: return fh.readlines() except Exception as e: return abort("Can't read %s: %s", short(path), e, fatal=(fatal, default))
def compress(source, destination, arcname=UNSET, ext=None, overwrite=True, fatal=True, logger=UNSET, dryrun=UNSET): """ Args: source (str | Path | None): Source folder to compress destination (str | Path | None): Destination folder arcname (str | None): Name of subfolder in archive (default: source basename) ext (str | None): Extension determining compression (default: extension of given 'source' file) overwrite (bool | None): True: replace existing, False: fail if destination exists, None: no destination check fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter dryrun (bool | UNSET | None): Optionally override current dryrun setting Returns: (int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed """ if not ext: _, _, ext = str(destination).lower().rpartition(".") kwargs = {} ext = SYS_INFO.platform_id.canonical_compress_extension(ext, short_form=True) if not ext: message = "Unknown extension '%s': can't compress file" % os.path.basename(destination) return abort(message, return_value=-1, fatal=fatal, logger=logger) if arcname is UNSET: arcname = os.path.basename(source) arcname = to_path(arcname or "") if ext == "zip": func = _zip else: func = _tar kwargs["mode"] = "w:" if ext == "tar" else "w:%s" % ext return _file_op(source, destination, func, overwrite, fatal, logger, dryrun, arcname=arcname, **kwargs)
def _file_op(source, destination, func, adapter, fatal, logger, must_exist=True): """ Call func(source, destination) :param str|None source: Source file or folder :param str|None destination: Destination file or folder :param callable func: Implementation function :param callable adapter: Optional function to call on 'source' before copy :param bool|None fatal: Abort execution on failure if True :param callable|None logger: Logger to use :param bool must_exist: If True, verify that source does indeed exist :return int: 1 if effectively done, 0 if no-op, -1 on failure """ if not source or not destination or source == destination: return 0 action = func.__name__[1:] indicator = "<-" if action == "symlink" else "->" psource = parent_folder(source) pdest = resolved_path(destination) if psource != pdest and psource.startswith(pdest): return abort("Can't %s %s %s %s: source contained in destination", action, short(source), indicator, short(destination), fatal=(fatal, -1)) if is_dryrun(): LOG.debug("Would %s %s %s %s", action, short(source), indicator, short(destination)) return 1 if must_exist and not os.path.exists(source): return abort("%s does not exist, can't %s to %s", short(source), action.title(), short(destination), fatal=(fatal, -1)) try: # Delete destination, but ensure that its parent folder exists delete(destination, fatal=fatal, logger=None) ensure_folder(destination, fatal=fatal, logger=None) if logger: note = adapter(source, destination, fatal=fatal, logger=logger) if adapter else "" if logger: logger("%s %s %s %s%s", action.title(), short(source), indicator, short(destination), note) func(source, destination) return 1 except Exception as e: return abort("Can't %s %s %s %s: %s", action, short(source), indicator, short(destination), e, fatal=(fatal, -1))
def run(program, *args, background=False, fatal=True, logger=UNSET, dryrun=UNSET, short_exe=UNSET, passthrough=False, path_env=None, strip="\r\n", stdout=subprocess.PIPE, stderr=subprocess.PIPE, **popen_args): """Run 'program' with 'args' Args: program (str | pathlib.Path): Program to run (full path, or basename) *args: Command line args to call 'program' with background (bool): When True, background the spawned process (detach from console and current process) fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter dryrun (bool | UNSET | None): Optionally override current dryrun setting short_exe (str | bool | None): Try to log a compact representation of executable passthrough (bool | file | None): If True-ish, pass-through stderr/stdout in addition to capturing it as well as 'passthrough' itself if it has a write() function path_env (dict | None): Allows to inject PATH-like env vars, see `_added_env_paths()` strip (str | bool | None): If provided, `strip()` the captured output [default: strip "\n" newlines] stdout (int | IO[Any] | None): Passed-through to subprocess.Popen, [default: subprocess.PIPE] stderr (int | IO[Any] | None): Passed-through to subprocess.Popen, [default: subprocess.PIPE] **popen_args: Passed through to `subprocess.Popen` Returns: (RunResult): Run outcome, use .failed, .succeeded, .output, .error etc to inspect the outcome """ if path_env: popen_args["env"] = _added_env_paths(path_env, env=popen_args.get("env")) args = flattened(args, shellify=True) full_path = which(program) result = RunResult(audit=RunAudit(full_path or program, args, popen_args)) description = result.audit.run_description(short_exe=short_exe) if background: description += " &" abort_logger = None if logger is None else UNSET if logger is True or logger is print: # When logger is True, we just print() the message, so we may as well color it nicely description = _R.colored(description, "bold") if _R.hdry(dryrun, logger, "run: %s" % description): result.audit.dryrun = True result.exit_code = 0 if stdout is not None: result.output = "[dryrun] %s" % description # Properly simulate a successful run if stdout is not None: result.error = "" return result if not full_path: if program and os.path.basename(program) == program: result.error = "%s is not installed (PATH=%s)" % ( short(program), short(os.environ.get("PATH"))) else: result.error = "%s is not an executable" % short(program) return abort(result.error, return_value=result, fatal=fatal, logger=abort_logger) _R.hlog(logger, "Running: %s" % description) if background: child_pid = daemonize() if child_pid: result.pid = child_pid # In parent process, we just report a successful run (we don't wait/check on background process) result.exit_code = 0 return result fatal = False # pragma: no cover, non-fatal mode in background process (there is no more console etc to report anything) with _WrappedArgs([full_path] + args) as wrapped_args: try: p, out, err = _run_popen(wrapped_args, popen_args, passthrough, fatal, stdout, stderr) result.output = decode(out, strip=strip) result.error = decode(err, strip=strip) result.pid = p.pid result.exit_code = p.returncode except Exception as e: if fatal: # Don't re-wrap with an abort(), let original stacktrace show through raise result.exc_info = e if not result.error: result.error = "%s failed: %s" % ( short(program), repr(e) if isinstance(e, OSError) else e) if fatal and result.exit_code: base_message = "%s exited with code %s" % (short(program), result.exit_code) if passthrough and (result.output or result.error): exception = _R.abort_exception(override=fatal) if exception is SystemExit: raise SystemExit(result.exit_code) if isinstance(exception, type) and issubclass( exception, BaseException): raise exception(base_message) message = [] if abort_logger is not None and not passthrough: # Log full output, unless user explicitly turned it off message.append("Run failed: %s" % description) if result.error: message.append("\nstderr:\n%s" % result.error) if result.output: message.append("\nstdout:\n%s" % result.output) message.append(base_message) abort("\n".join(message), code=result.exit_code, exc_info=result.exc_info, fatal=fatal, logger=abort_logger) if background: os._exit( result.exit_code ) # pragma: no cover, simply exit forked process (don't go back to caller) return result
def _file_op(source, destination, func, overwrite, fatal, logger, dryrun, must_exist=True, ignore=None, **extra): """Call func(source, destination) Args: source (str | None): Source file or folder destination (str | None): Destination file or folder func (callable): Implementation function overwrite (bool | None): True: replace existing, False: fail if destination exists, None: no destination check fatal (type | bool | None): True: abort execution on failure, False: don't abort but log, None: don't abort, don't log logger (callable | bool | None): Logger to use, True to print(), False to trace(), None to disable log chatter dryrun (bool | UNSET | None): Optionally override current dryrun setting must_exist (bool): If True, verify that source does indeed exist ignore (callable | list | str | None): Names to be ignored **extra: Passed-through to 'func' Returns: (int): In non-fatal mode, 1: successfully done, 0: was no-op, -1: failed """ if not source or not destination or source == destination: return 0 action = func.__name__[1:] indicator = "<-" if action == "symlink" else "->" description = "%s %s %s %s" % (action, short(source), indicator, short(destination)) psource = parent_folder(source) pdest = resolved_path(destination) if psource.startswith(pdest): message = "Can't %s: source contained in destination" % description return abort(message, return_value=-1, fatal=fatal, logger=logger) if _R.hdry(dryrun, logger, description): return 1 if must_exist and not (os.path.exists(source) or os.path.islink(source)): message = "%s does not exist, can't %s to %s" % (short(source), action.lower(), short(destination)) return abort(message, return_value=-1, fatal=fatal, logger=logger) if overwrite is not None: islink = os.path.islink(pdest) if islink or os.path.exists(pdest): if not overwrite: message = "%s exists, can't %s" % (short(destination), action.lower()) return abort(message, return_value=-1, fatal=fatal, logger=logger) _do_delete(pdest, islink, fatal) try: # Ensure parent folder exists ensure_folder(parent_folder(destination), fatal=fatal, logger=None, dryrun=dryrun) _R.hlog(logger, "%s%s" % (description[0].upper(), description[1:])) if ignore is not None: if not callable(ignore): given = ignore def ignore(*_): return given extra["ignore"] = ignore func(source, destination, **extra) return 1 except Exception as e: return abort("Can't %s" % description, exc_info=e, return_value=-1, fatal=fatal, logger=logger)