def actions_for_dir_path(
path: str, path_acc: t.Set[str] = set()) -> t.List[Action]:
"""
Returns a list of actions that is needed to create a folder and it's parent folders.
:param path:
:param path_acc: paths already examined
"""
path = abspath(path)
typecheck_locals(path=FileName(allow_non_existent=False) | DirName(),
create=Bool())
assert os.path.exists(path)
if path == "" or path == "~":
return []
path = normalize_path(path)
parts = path.split("/")
ret = []
for i in range(
2 if parts[0] == "~" else 1,
len(parts) + 1 if os.path.isdir(abspath(path)) else len(parts)):
subpath = "/".join(parts[:i])
subpath_norm = normalize_path(subpath)
if subpath_norm in path_acc:
continue
ret.append(CreateDir(subpath_norm))
path_acc.add(subpath_norm)
return ret
def retrieve_file(self, key: Key, subkey: str, destination: str):
"""
Copies the stored file under the passed key to its new destination.
"""
destination = abspath(destination)
source = self._storage_filename(self[key, subkey])
shutil.copy(source, destination)
def matched_paths(base: str, include_patterns: t.Union[t.List['str'], str] = ["**", "**/.*"],
exclude_patterns: t.List[str] = None) -> t.List[str]:
"""
All matching paths in base directory and it's child directories.
:param base: base directory
:param include_patterns: patterns that match the paths that should be included
:param exclude_patterns: patterns that match the paths that should be excluded
:return: matching paths
"""
typecheck_locals(base=str, include_pattern=List(Str())|Str(), exclude_patterns=Optional(List(Str())))
if isinstance(include_patterns, list):
ret = []
for pattern in include_patterns:
ret.extend(matched_paths(base, pattern, exclude_patterns))
return ret
cwd = os.getcwd()
os.chdir(abspath(base))
import glob2, globster
names = glob2.glob(include_pattern)
if exclude_patterns:
exclude_globster = globster.Globster(exclude_patterns)
names = [x for x in names if not exclude_globster.match(x)]
names = list(map(abspath, names))
return names
def _execute(self, db: Database):
logging.info("Execute {!r} in directory {!r}".format(
self.cmd, self.working_dir))
proc = subprocess.Popen(["/bin/sh", "-c", self.cmd],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
cwd=abspath(self.working_dir))
out, err = proc.communicate()
ret_code = proc.wait()
if self.send_mail:
def format_out(out):
return out.replace("\n", "\n\t")
content = """
Command: {cmd}
Return code: {ret}
Standard out:
{out}
Standard Error:
{err}
""".format(cmd=self.cmd,
ret=ret_code,
out=format_out(out),
err=format_out(err))
send_mail(self.mail_address, self.mail_header, content)
def matched_paths(base: str,
include_patterns: t.Union[t.List['str'],
str] = ["**", "**/.*"],
exclude_patterns: t.List[str] = None) -> t.List[str]:
"""
All matching paths in base directory and it's child directories.
:param base: base directory
:param include_patterns: patterns that match the paths that should be included
:param exclude_patterns: patterns that match the paths that should be excluded
:return: matching paths
"""
typecheck_locals(base=str,
include_pattern=List(Str()) | Str(),
exclude_patterns=Optional(List(Str())))
if isinstance(include_patterns, list):
ret = []
for pattern in include_patterns:
ret.extend(matched_paths(base, pattern, exclude_patterns))
return ret
cwd = os.getcwd()
os.chdir(abspath(base))
import glob2, globster
names = glob2.glob(include_pattern)
if exclude_patterns:
exclude_globster = globster.Globster(exclude_patterns)
names = [x for x in names if not exclude_globster.match(x)]
names = list(map(abspath, names))
return names
def retrieve_file(self, key: Key, subkey: str, destination: str):
"""
Copies the stored file under the passed key to its new destination.
"""
destination = abspath(destination)
source = self._storage_filename(self[key, subkey])
shutil.copy(source, destination)
def load(self, filename: str):
"""
Cleans the database and then loads the database from the compressed archive.
:param filename: name of the compressed archive
"""
os.system("tar xf '{file}' -C '{dest}'".format(file=abspath(filename), dest=self.tmp_dir))
self._data.clear()
self._load_yaml()
def load(self, filename: str):
"""
Cleans the database and then loads the database from the compressed archive.
:param filename: name of the compressed archive
"""
os.system("tar xf '{file}' -C '{dest}'".format(file=abspath(filename),
dest=self.tmp_dir))
self._data.clear()
self._load_yaml()
def actions_for_dir_path(path: str, path_acc: t.Set[str] = set()) -> t.List[Action]:
"""
Returns a list of actions that is needed to create a folder and it's parent folders.
:param path:
:param path_acc: paths already examined
"""
path = abspath(path)
typecheck_locals(path=FileName(allow_non_existent=False)|DirName(), create=Bool())
assert os.path.exists(path)
if path == "" or path == "~":
return []
path = normalize_path(path)
parts = path.split("/")
ret = []
for i in range(2 if parts[0] == "~" else 1, len(parts) + 1 if os.path.isdir(abspath(path)) else len(parts)):
subpath = "/".join(parts[:i])
subpath_norm = normalize_path(subpath)
if subpath_norm in path_acc:
continue
ret.append(CreateDir(subpath_norm))
path_acc.add(subpath_norm)
return ret
def store_file(self, key: Key, subkey: str, file_path: str):
"""
Stores a file in the database and stores the files id under the passed key as an string.
It uses the hashed (sha512 + md5) file contents as the id. A file can't be deleted by storing
another file under the same key.
:param key: passed key
:param file_path: path of the file to store
"""
file_path = abspath(file_path)
typecheck_locals(file_path=FileName())
file_id = hashed_name_of_file(file_path)
shutil.copy(file_path, self._storage_filename(file_id))
self[key, subkey] = file_id
def store_file(self, key: Key, subkey: str, file_path: str):
"""
Stores a file in the database and stores the files id under the passed key as an string.
It uses the hashed (sha512 + md5) file contents as the id. A file can't be deleted by storing
another file under the same key.
:param key: passed key
:param file_path: path of the file to store
"""
file_path = abspath(file_path)
typecheck_locals(file_path=FileName())
file_id = hashed_name_of_file(file_path)
shutil.copy(file_path, self._storage_filename(file_id))
self[key, subkey] = file_id
def store(self, filename: str, compression_level: int = None):
"""
Store the whole database as a compressed archive under the given file name.
:param filename: passed file name
:param compression_level: used compression level, from -1 (low) to -9 (high)
"""
compression_level = compression_level or Settings()["package/compression/level"]
self._store_yaml()
filename = abspath(filename)
used_prog = "gzip"
av_programs = ["pixz", "xz"] if Settings()["package/compression/program"] == "xz" else ["pigz", "gzip"]
for prog in av_programs:
if does_command_succeed(prog + " --version"):
used_prog = prog
break
cmd = "cd {dir}; XZ={l} GZIP={l} tar cf '{dest}' . --use-compress-program={prog}"\
.format(l=compression_level, dest=filename, dir=self.tmp_dir, prog=used_prog)
res = subprocess.check_output(["/bin/sh", "-c", cmd])
def _execute(self, db: Database):
logging.info("Execute {!r} in directory {!r}".format(self.cmd, self.working_dir))
proc = subprocess.Popen(["/bin/sh", "-c", self.cmd],
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=True, cwd=abspath(self.working_dir))
out, err = proc.communicate()
ret_code = proc.wait()
if self.send_mail:
def format_out(out):
return out.replace("\n", "\n\t")
content = """
Command: {cmd}
Return code: {ret}
Standard out:
{out}
Standard Error:
{err}
""".format(cmd=self.cmd, ret=ret_code, out=format_out(out), err=format_out(err))
send_mail(self.mail_address, self.mail_header, content)
def store(self, filename: str, compression_level: int = None):
"""
Store the whole database as a compressed archive under the given file name.
:param filename: passed file name
:param compression_level: used compression level, from -1 (low) to -9 (high)
"""
compression_level = compression_level or Settings(
)["package/compression/level"]
self._store_yaml()
filename = abspath(filename)
used_prog = "gzip"
av_programs = ["pixz", "xz"] if Settings(
)["package/compression/program"] == "xz" else ["pigz", "gzip"]
for prog in av_programs:
if does_command_succeed(prog + " --version"):
used_prog = prog
break
cmd = "cd {dir}; XZ={l} GZIP={l} tar cf '{dest}' . --use-compress-program={prog}"\
.format(l=compression_level, dest=filename, dir=self.tmp_dir, prog=used_prog)
res = subprocess.check_output(["/bin/sh", "-c", cmd])
def _dry_run_message(self) -> str:
verb = "override" if os.path.exists(abspath(self.dest)) else "create"
return "Would {} file {!r}".format(verb, self.dest)
def _execute(self, db: Database):
shutil.rmtree(abspath(self.directory))
def reverse(self) -> t.List['RemoveDir']:
if os.path.exists(abspath(self.directory)):
return []
return [RemoveDir(self.directory)]
def _execute(self, db: Database):
if not os.path.exists(abspath(self.directory)):
os.mkdir(abspath(self.directory))
def _dry_run_message(self) -> str:
if not os.path.exists(abspath(self.directory)):
return "Would create directory {!r}".format(self.directory)
return "Wouldn't create directory {!r} as it already exists".format(self.directory)
def _execute(self, db: Database):
if os.path.exists(self.file):
os.remove(abspath(self.file))
def reverse(self) -> t.List[Action]:
if os.path.exists(abspath(self.dest)):
return [CopyFile(self.dest)]
return [RemoveFile(self.dest)]
def _dry_run_message(self) -> str:
verb = "override" if os.path.exists(abspath(self.dest)) else "create"
return "Would {} file {!r}".format(verb, self.dest)
def reverse(self) -> t.List[Action]:
if os.path.exists(abspath(self.dest)):
return [CopyFile(self.dest)]
return [RemoveFile(self.dest)]
def _execute(self, db: Database):
if os.path.exists(self.file):
os.remove(abspath(self.file))
def _dry_run_message(self) -> str:
if not os.path.exists(abspath(self.directory)):
return "Would create directory {!r}".format(self.directory)
return "Wouldn't create directory {!r} as it already exists".format(
self.directory)
def _execute(self, db: Database):
if not os.path.exists(abspath(self.directory)):
os.mkdir(abspath(self.directory))
def reverse(self) -> t.List['RemoveDir']:
if os.path.exists(abspath(self.directory)):
return []
return [RemoveDir(self.directory)]
def _execute(self, db: Database):
shutil.rmtree(abspath(self.directory))
