def process_one_dir(self, location, subdirpath, depth, results): ''' arguments: location: the location being checked subdirpath: the path to the subdirectory being checked depth: the depth of the directory being checked (starting at 1) results: the result set ''' if self.dirs_to_check is not None: if not dirtree_check(subdirpath, self.dirs_to_check): return if clouseau.retention.utils.ignores.dir_is_ignored(subdirpath, self.ignored): return True count = 0 # doing a directory higher up in the tree than our depth cutoff, # only do the files in it, because we have the full list of dirs # up to our cutoff we do them one by one if depth < self.depth: filenames = os.listdir(subdirpath) files = [] for fname in filenames: try: filestat = os.stat(os.path.join(subdirpath, fname)) except: continue if (not stat.S_ISLNK(filestat.st_mode) and stat.S_ISREG(filestat.st_mode)): files.append((fname, filestat)) self.process_files_from_path(location, subdirpath, files, count, results) return # doing a directory at our cutoff depth, walk it, # because anything below the depth # cutoff won't be in our list temp_results = [] for base, paths, files in self.walk_nolinks(subdirpath): expanded_dirs, wildcard_dirs = expand_ignored_dirs(base, self.ignored) if self.dirs_to_check is not None: paths[:] = [p for p in paths if dirtree_check(os.path.join(base, p), self.dirs_to_check)] paths[:] = [p for p in paths if (not clouseau.retention.utils.fileutils.startswithpath(os.path.join( base, p), expanded_dirs) and not clouseau.retention.utils.fileutils.wildcard_matches(os.path.join( base, p), wildcard_dirs, exact=False))] count = self.process_files_from_path(location, base, files, count, temp_results) if count > self.max_files: return results.extend(temp_results)
def get_subdirs_to_do(self, dirname, dirname_depth, todo): locale.setlocale(locale.LC_ALL, '') if clouseau.retention.utils.ignores.dir_is_ignored(dirname, self.ignored): return todo if clouseau.retention.utils.fileutils.dir_is_wrong_type(dirname): return todo if self.depth < dirname_depth: return todo if dirname_depth not in todo: todo[dirname_depth] = [] if self.dirs_to_check is not None: if clouseau.retention.utils.fileutils.subdir_check(dirname, self.dirs_to_check): todo[dirname_depth].append(dirname) else: todo[dirname_depth].append(dirname) if self.depth == dirname_depth: # don't read below the depth level return todo dirs = [os.path.join(dirname, d) for d in os.listdir(dirname)] if self.dirs_to_check is not None: dirs = [d for d in dirs if dirtree_check(d, self.dirs_to_check)] for dname in dirs: todo = self.get_subdirs_to_do(dname, dirname_depth + 1, todo) return todo
def get_subdirs_to_do(self, dirname, dirname_depth, todo): locale.setlocale(locale.LC_ALL, '') if clouseau.retention.utils.ignores.dir_is_ignored( dirname, self.ignored): return todo if clouseau.retention.utils.fileutils.dir_is_wrong_type(dirname): return todo if self.depth < dirname_depth: return todo if dirname_depth not in todo: todo[dirname_depth] = [] if self.dirs_to_check is not None: if clouseau.retention.utils.fileutils.subdir_check( dirname, self.dirs_to_check): todo[dirname_depth].append(dirname) else: todo[dirname_depth].append(dirname) if self.depth == dirname_depth: # don't read below the depth level return todo dirs = [os.path.join(dirname, d) for d in os.listdir(dirname)] if self.dirs_to_check is not None: dirs = [d for d in dirs if dirtree_check(d, self.dirs_to_check)] for dname in dirs: todo = self.get_subdirs_to_do(dname, dirname_depth + 1, todo) return todo
def get_dirs_to_do(self, dirname): if (self.dirs_to_check is not None and not dirtree_check(dirname, self.dirs_to_check)): return {} todo = {} depth_of_dirname = dirname.count(os.path.sep) todo = self.get_subdirs_to_do(dirname, depth_of_dirname, todo) return todo
def get_dirs_to_do(self, dirname): if (self.dirs_to_check is not None and not dirtree_check(dirname, self.dirs_to_check)): return {} todo = {} depth_of_dirname = dirname.count(os.path.sep) todo = self.get_subdirs_to_do(dirname, depth_of_dirname, todo) return todo
def process_one_dir(self, location, subdirpath, depth, results): ''' arguments: location: the location being checked subdirpath: the path to the subdirectory being checked depth: the depth of the directory being checked (starting at 1) results: the result set ''' if self.dirs_to_check is not None: if not dirtree_check(subdirpath, self.dirs_to_check): return if clouseau.retention.utils.ignores.dir_is_ignored( subdirpath, self.ignored): return True count = 0 # doing a directory higher up in the tree than our depth cutoff, # only do the files in it, because we have the full list of dirs # up to our cutoff we do them one by one if depth < self.depth: filenames = os.listdir(subdirpath) files = [] for fname in filenames: try: filestat = os.stat(os.path.join(subdirpath, fname)) except: continue if (not stat.S_ISLNK(filestat.st_mode) and stat.S_ISREG(filestat.st_mode)): files.append((fname, filestat)) self.process_files_from_path(location, subdirpath, files, count, results) return # doing a directory at our cutoff depth, walk it, # because anything below the depth # cutoff won't be in our list temp_results = [] for base, paths, files in self.walk_nolinks(subdirpath): expanded_dirs, wildcard_dirs = expand_ignored_dirs( base, self.ignored) if self.dirs_to_check is not None: paths[:] = [ p for p in paths if dirtree_check(os.path.join(base, p), self.dirs_to_check) ] paths[:] = [ p for p in paths if (not clouseau.retention.utils.fileutils.startswithpath( os.path.join(base, p), expanded_dirs) and not clouseau.retention.utils.fileutils.wildcard_matches( os.path.join(base, p), wildcard_dirs, exact=False)) ] count = self.process_files_from_path(location, base, files, count, temp_results) if count > self.max_files: return results.extend(temp_results)