def actions_for_dir_path( path: str, path_acc: t.Set[str] = set()) -> t.List[Action]: """ Returns a list of actions that is needed to create a folder and it's parent folders. :param path: :param path_acc: paths already examined """ path = abspath(path) typecheck_locals(path=FileName(allow_non_existent=False) | DirName(), create=Bool()) assert os.path.exists(path) if path == "" or path == "~": return [] path = normalize_path(path) parts = path.split("/") ret = [] for i in range( 2 if parts[0] == "~" else 1, len(parts) + 1 if os.path.isdir(abspath(path)) else len(parts)): subpath = "/".join(parts[:i]) subpath_norm = normalize_path(subpath) if subpath_norm in path_acc: continue ret.append(CreateDir(subpath_norm)) path_acc.add(subpath_norm) return ret
def retrieve_file(self, key: Key, subkey: str, destination: str): """ Copies the stored file under the passed key to its new destination. """ destination = abspath(destination) source = self._storage_filename(self[key, subkey]) shutil.copy(source, destination)
def matched_paths(base: str, include_patterns: t.Union[t.List['str'], str] = ["**", "**/.*"], exclude_patterns: t.List[str] = None) -> t.List[str]: """ All matching paths in base directory and it's child directories. :param base: base directory :param include_patterns: patterns that match the paths that should be included :param exclude_patterns: patterns that match the paths that should be excluded :return: matching paths """ typecheck_locals(base=str, include_pattern=List(Str())|Str(), exclude_patterns=Optional(List(Str()))) if isinstance(include_patterns, list): ret = [] for pattern in include_patterns: ret.extend(matched_paths(base, pattern, exclude_patterns)) return ret cwd = os.getcwd() os.chdir(abspath(base)) import glob2, globster names = glob2.glob(include_pattern) if exclude_patterns: exclude_globster = globster.Globster(exclude_patterns) names = [x for x in names if not exclude_globster.match(x)] names = list(map(abspath, names)) return names
def _execute(self, db: Database): logging.info("Execute {!r} in directory {!r}".format( self.cmd, self.working_dir)) proc = subprocess.Popen(["/bin/sh", "-c", self.cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, cwd=abspath(self.working_dir)) out, err = proc.communicate() ret_code = proc.wait() if self.send_mail: def format_out(out): return out.replace("\n", "\n\t") content = """ Command: {cmd} Return code: {ret} Standard out: {out} Standard Error: {err} """.format(cmd=self.cmd, ret=ret_code, out=format_out(out), err=format_out(err)) send_mail(self.mail_address, self.mail_header, content)
def matched_paths(base: str, include_patterns: t.Union[t.List['str'], str] = ["**", "**/.*"], exclude_patterns: t.List[str] = None) -> t.List[str]: """ All matching paths in base directory and it's child directories. :param base: base directory :param include_patterns: patterns that match the paths that should be included :param exclude_patterns: patterns that match the paths that should be excluded :return: matching paths """ typecheck_locals(base=str, include_pattern=List(Str()) | Str(), exclude_patterns=Optional(List(Str()))) if isinstance(include_patterns, list): ret = [] for pattern in include_patterns: ret.extend(matched_paths(base, pattern, exclude_patterns)) return ret cwd = os.getcwd() os.chdir(abspath(base)) import glob2, globster names = glob2.glob(include_pattern) if exclude_patterns: exclude_globster = globster.Globster(exclude_patterns) names = [x for x in names if not exclude_globster.match(x)] names = list(map(abspath, names)) return names
def load(self, filename: str): """ Cleans the database and then loads the database from the compressed archive. :param filename: name of the compressed archive """ os.system("tar xf '{file}' -C '{dest}'".format(file=abspath(filename), dest=self.tmp_dir)) self._data.clear() self._load_yaml()
def actions_for_dir_path(path: str, path_acc: t.Set[str] = set()) -> t.List[Action]: """ Returns a list of actions that is needed to create a folder and it's parent folders. :param path: :param path_acc: paths already examined """ path = abspath(path) typecheck_locals(path=FileName(allow_non_existent=False)|DirName(), create=Bool()) assert os.path.exists(path) if path == "" or path == "~": return [] path = normalize_path(path) parts = path.split("/") ret = [] for i in range(2 if parts[0] == "~" else 1, len(parts) + 1 if os.path.isdir(abspath(path)) else len(parts)): subpath = "/".join(parts[:i]) subpath_norm = normalize_path(subpath) if subpath_norm in path_acc: continue ret.append(CreateDir(subpath_norm)) path_acc.add(subpath_norm) return ret
def store_file(self, key: Key, subkey: str, file_path: str): """ Stores a file in the database and stores the files id under the passed key as an string. It uses the hashed (sha512 + md5) file contents as the id. A file can't be deleted by storing another file under the same key. :param key: passed key :param file_path: path of the file to store """ file_path = abspath(file_path) typecheck_locals(file_path=FileName()) file_id = hashed_name_of_file(file_path) shutil.copy(file_path, self._storage_filename(file_id)) self[key, subkey] = file_id
def store(self, filename: str, compression_level: int = None): """ Store the whole database as a compressed archive under the given file name. :param filename: passed file name :param compression_level: used compression level, from -1 (low) to -9 (high) """ compression_level = compression_level or Settings()["package/compression/level"] self._store_yaml() filename = abspath(filename) used_prog = "gzip" av_programs = ["pixz", "xz"] if Settings()["package/compression/program"] == "xz" else ["pigz", "gzip"] for prog in av_programs: if does_command_succeed(prog + " --version"): used_prog = prog break cmd = "cd {dir}; XZ={l} GZIP={l} tar cf '{dest}' . --use-compress-program={prog}"\ .format(l=compression_level, dest=filename, dir=self.tmp_dir, prog=used_prog) res = subprocess.check_output(["/bin/sh", "-c", cmd])
def _execute(self, db: Database): logging.info("Execute {!r} in directory {!r}".format(self.cmd, self.working_dir)) proc = subprocess.Popen(["/bin/sh", "-c", self.cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, cwd=abspath(self.working_dir)) out, err = proc.communicate() ret_code = proc.wait() if self.send_mail: def format_out(out): return out.replace("\n", "\n\t") content = """ Command: {cmd} Return code: {ret} Standard out: {out} Standard Error: {err} """.format(cmd=self.cmd, ret=ret_code, out=format_out(out), err=format_out(err)) send_mail(self.mail_address, self.mail_header, content)
def store(self, filename: str, compression_level: int = None): """ Store the whole database as a compressed archive under the given file name. :param filename: passed file name :param compression_level: used compression level, from -1 (low) to -9 (high) """ compression_level = compression_level or Settings( )["package/compression/level"] self._store_yaml() filename = abspath(filename) used_prog = "gzip" av_programs = ["pixz", "xz"] if Settings( )["package/compression/program"] == "xz" else ["pigz", "gzip"] for prog in av_programs: if does_command_succeed(prog + " --version"): used_prog = prog break cmd = "cd {dir}; XZ={l} GZIP={l} tar cf '{dest}' . --use-compress-program={prog}"\ .format(l=compression_level, dest=filename, dir=self.tmp_dir, prog=used_prog) res = subprocess.check_output(["/bin/sh", "-c", cmd])
def _dry_run_message(self) -> str: verb = "override" if os.path.exists(abspath(self.dest)) else "create" return "Would {} file {!r}".format(verb, self.dest)
def _execute(self, db: Database): shutil.rmtree(abspath(self.directory))
def reverse(self) -> t.List['RemoveDir']: if os.path.exists(abspath(self.directory)): return [] return [RemoveDir(self.directory)]
def _execute(self, db: Database): if not os.path.exists(abspath(self.directory)): os.mkdir(abspath(self.directory))
def _dry_run_message(self) -> str: if not os.path.exists(abspath(self.directory)): return "Would create directory {!r}".format(self.directory) return "Wouldn't create directory {!r} as it already exists".format(self.directory)
def _execute(self, db: Database): if os.path.exists(self.file): os.remove(abspath(self.file))
def reverse(self) -> t.List[Action]: if os.path.exists(abspath(self.dest)): return [CopyFile(self.dest)] return [RemoveFile(self.dest)]
def _dry_run_message(self) -> str: if not os.path.exists(abspath(self.directory)): return "Would create directory {!r}".format(self.directory) return "Wouldn't create directory {!r} as it already exists".format( self.directory)