def write(path, contents, fatal=True, logger=None):
"""
:param str|None path: Path to file
:param str|None contents: Contents to write
:param bool|None fatal: Abort execution on failure if True
:param callable|None logger: Logger to use
:return int: 1 if effectively done, 0 if no-op, -1 on failure
"""
if not path:
return 0
if is_dryrun():
action = "write %s bytes to" % len(contents) if contents else "touch"
LOG.debug("Would %s %s", action, short(path))
return 1
ensure_folder(path, fatal=fatal, logger=logger)
if logger and contents:
logger("Writing %s bytes to %s", len(contents), short(path))
try:
with io.open(path, "wt") as fh:
if contents:
fh.write(decode(contents))
else:
os.utime(path, None)
return 1
except Exception as e:
return abort("Can't write to %s: %s",
short(path),
e,
fatal=(fatal, -1))
def delete(path, fatal=True, logger=LOG.debug):
"""
:param str|None path: Path to file or folder to delete
:param bool|None fatal: Abort execution on failure if True
:param callable|None logger: Logger to use
:return int: 1 if effectively done, 0 if no-op, -1 on failure
"""
islink = path and os.path.islink(path)
if not islink and (not path or not os.path.exists(path)):
return 0
if is_dryrun():
LOG.debug("Would delete %s", short(path))
return 1
if logger:
logger("Deleting %s", short(path))
try:
if islink or os.path.isfile(path):
os.unlink(path)
else:
shutil.rmtree(path)
return 1
except Exception as e:
return abort("Can't delete %s: %s", short(path), e, fatal=(fatal, -1))
def read_json(path, default=None, fatal=True, logger=None):
"""
:param str|None path: Path to file to deserialize
:param dict|list|None default: Default if file is not present, or if it's not json
:param bool|None fatal: Abort execution on failure if True
:param callable|None logger: Logger to use
:return dict|list: Deserialized data from file
"""
path = resolved_path(path)
if not path or not os.path.exists(path):
if default is None:
return abort("No file %s", short(path), fatal=(fatal, default))
return default
try:
with io.open(path, "rt") as fh:
data = json.load(fh)
if default is not None and type(data) != type(default):
return abort("Wrong type %s for %s, expecting %s", type(data), short(path), type(default), fatal=(fatal, default))
if logger:
logger("Read %s", short(path))
return data
except Exception as e:
return abort("Couldn't read %s: %s", short(path), e, fatal=(fatal, default))
def run(program, *args, **kwargs):
"""Run 'program' with 'args'"""
args = flattened(args, split=SHELL)
full_path = which(program)
logger = kwargs.pop("logger", LOG.debug)
fatal = kwargs.pop("fatal", True)
dryrun = kwargs.pop("dryrun", is_dryrun())
include_error = kwargs.pop("include_error", False)
message = "Would run" if dryrun else "Running"
message = "%s: %s %s" % (message, short(
full_path or program), represented_args(args))
if logger:
logger(message)
if dryrun:
return message
if not full_path:
return abort("%s is not installed", short(program), fatal=fatal)
stdout = kwargs.pop("stdout", subprocess.PIPE)
stderr = kwargs.pop("stderr", subprocess.PIPE)
args = [full_path] + args
try:
path_env = kwargs.pop("path_env", None)
if path_env:
kwargs["env"] = added_env_paths(path_env, env=kwargs.get("env"))
p = subprocess.Popen(args, stdout=stdout, stderr=stderr,
**kwargs) # nosec
output, err = p.communicate()
output = decode(output, strip=True)
err = decode(err, strip=True)
if p.returncode and fatal is not None:
note = ": %s\n%s" % (err, output) if output or err else ""
message = "%s exited with code %s%s" % (short(program),
p.returncode, note.strip())
return abort(message, fatal=fatal)
if include_error and err:
output = "%s\n%s" % (output, err)
return output and output.strip()
except Exception as e:
return abort("%s failed: %s",
short(program),
e,
exc_info=e,
fatal=fatal)
def ensure_folder(path,
folder=False,
fatal=True,
logger=LOG.debug,
dryrun=None):
"""
:param str|None path: Path to file or folder
:param bool folder: If True, 'path' refers to a folder (file otherwise)
:param bool|None fatal: Abort execution on failure if True
:param callable|None logger: Logger to use
:param bool|None dryrun: If specified, override global is_dryrun()
:return int: 1 if effectively done, 0 if no-op, -1 on failure
"""
if not path:
return 0
if folder:
folder = resolved_path(path)
else:
folder = parent_folder(path)
if os.path.isdir(folder):
if not os.access(folder, os.W_OK):
return abort("Folder %s is not writable",
folder,
fatal=(fatal, -1),
logger=logger)
return 0
if dryrun is None:
dryrun = is_dryrun()
if dryrun:
LOG.debug("Would create %s", short(folder))
return 1
try:
os.makedirs(folder)
if logger:
logger("Created folder %s", short(folder))
return 1
except Exception as e:
return abort("Can't create folder %s: %s",
short(folder),
e,
fatal=(fatal, -1),
logger=logger)
def save_json(data, path, fatal=True, logger=None, sort_keys=True, indent=2, **kwargs):
"""
Args:
data (object | None): Data to serialize and save
path (str | None): Path to file where to save
fatal (bool | None): Abort execution on failure if True
logger (callable | None): Logger to use
sort_keys (bool): Save json with sorted keys
indent (int): Indentation to use
**kwargs: Passed through to `json.dump()`
Returns:
(int): 1 if saved, -1 if failed (when `fatal` is False)
"""
if data is None or not path:
return abort("No file %s", short(path), fatal=fatal)
try:
path = resolved_path(path)
ensure_folder(path, fatal=fatal, logger=None)
if is_dryrun():
LOG.info("Would save %s", short(path))
return 1
if hasattr(data, "to_dict"):
data = data.to_dict()
if indent:
kwargs.setdefault("separators", (",", ': '))
with open(path, "wt") as fh:
json.dump(data, fh, sort_keys=sort_keys, indent=indent, **kwargs)
fh.write("\n")
if logger:
logger("Saved %s", short(path))
return 1
except Exception as e:
return abort("Couldn't save %s: %s", short(path), e, fatal=(fatal, -1))
def make_executable(path, fatal=True):
"""
:param str|None path: chmod file with 'path' as executable
:param bool|None fatal: Abort execution on failure if True
:return int: 1 if effectively done, 0 if no-op, -1 on failure
"""
if is_executable(path):
return 0
if is_dryrun():
LOG.debug("Would make %s executable", short(path))
return 1
if not os.path.exists(path):
return abort("%s does not exist, can't make it executable",
short(path),
fatal=(fatal, -1))
try:
os.chmod(path, 0o755) # nosec
return 1
except Exception as e:
return abort("Can't chmod %s: %s", short(path), e, fatal=(fatal, -1))
def load(self, path=None, fatal=True, logger=None):
"""
:param str|None path: Load this object from file with 'path' (default: self._path)
:param bool|None fatal: Abort execution on failure if True
:param callable|None logger: Logger to use
"""
self.reset()
if path:
self._path = path
self._source = short(path)
else:
path = getattr(self, "_path", None)
if path:
self.set_from_dict(read_json(path, default={}, fatal=fatal, logger=logger))
def get_lines(path, max_size=TEXT_THRESHOLD_SIZE, fatal=True, default=None):
"""
:param str|None path: Path of text file to return lines from
:param int|None max_size: Return contents only for files smaller than 'max_size' bytes
:param bool|None fatal: Abort execution on failure if True
:param list|None default: Object to return if lines couldn't be read
:return list|None: Lines from file contents
"""
if not path or not os.path.isfile(path) or (
max_size and os.path.getsize(path) > max_size):
# Intended for small text files, pretend no contents for binaries
return default
try:
with io.open(path, "rt", errors="ignore") as fh:
return fh.readlines()
except Exception as e:
return abort("Can't read %s: %s",
short(path),
e,
fatal=(fatal, default))
def _file_op(source,
destination,
func,
adapter,
fatal,
logger,
must_exist=True):
"""
Call func(source, destination)
:param str|None source: Source file or folder
:param str|None destination: Destination file or folder
:param callable func: Implementation function
:param callable adapter: Optional function to call on 'source' before copy
:param bool|None fatal: Abort execution on failure if True
:param callable|None logger: Logger to use
:param bool must_exist: If True, verify that source does indeed exist
:return int: 1 if effectively done, 0 if no-op, -1 on failure
"""
if not source or not destination or source == destination:
return 0
action = func.__name__[1:]
indicator = "<-" if action == "symlink" else "->"
psource = parent_folder(source)
pdest = resolved_path(destination)
if psource != pdest and psource.startswith(pdest):
return abort("Can't %s %s %s %s: source contained in destination",
action,
short(source),
indicator,
short(destination),
fatal=(fatal, -1))
if is_dryrun():
LOG.debug("Would %s %s %s %s", action, short(source), indicator,
short(destination))
return 1
if must_exist and not os.path.exists(source):
return abort("%s does not exist, can't %s to %s",
short(source),
action.title(),
short(destination),
fatal=(fatal, -1))
try:
# Delete destination, but ensure that its parent folder exists
delete(destination, fatal=fatal, logger=None)
ensure_folder(destination, fatal=fatal, logger=None)
if logger:
note = adapter(source, destination, fatal=fatal,
logger=logger) if adapter else ""
if logger:
logger("%s %s %s %s%s", action.title(), short(source),
indicator, short(destination), note)
func(source, destination)
return 1
except Exception as e:
return abort("Can't %s %s %s %s: %s",
action,
short(source),
indicator,
short(destination),
e,
fatal=(fatal, -1))