def process_one_dir(self, location, subdirpath, depth, results):
'''
arguments:
location: the location being checked
subdirpath: the path to the subdirectory being checked
depth: the depth of the directory being checked (starting at 1)
results: the result set
'''
if self.dirs_to_check is not None:
if not dirtree_check(subdirpath, self.dirs_to_check):
return
if clouseau.retention.utils.ignores.dir_is_ignored(subdirpath, self.ignored):
return True
count = 0
# doing a directory higher up in the tree than our depth cutoff,
# only do the files in it, because we have the full list of dirs
# up to our cutoff we do them one by one
if depth < self.depth:
filenames = os.listdir(subdirpath)
files = []
for fname in filenames:
try:
filestat = os.stat(os.path.join(subdirpath, fname))
except:
continue
if (not stat.S_ISLNK(filestat.st_mode) and
stat.S_ISREG(filestat.st_mode)):
files.append((fname, filestat))
self.process_files_from_path(location, subdirpath,
files, count, results)
return
# doing a directory at our cutoff depth, walk it,
# because anything below the depth
# cutoff won't be in our list
temp_results = []
for base, paths, files in self.walk_nolinks(subdirpath):
expanded_dirs, wildcard_dirs = expand_ignored_dirs(base, self.ignored)
if self.dirs_to_check is not None:
paths[:] = [p for p in paths
if dirtree_check(os.path.join(base, p), self.dirs_to_check)]
paths[:] = [p for p in paths if
(not clouseau.retention.utils.fileutils.startswithpath(os.path.join(
base, p), expanded_dirs) and
not clouseau.retention.utils.fileutils.wildcard_matches(os.path.join(
base, p), wildcard_dirs, exact=False))]
count = self.process_files_from_path(location, base, files,
count, temp_results)
if count > self.max_files:
return
results.extend(temp_results)
def get_subdirs_to_do(self, dirname, dirname_depth, todo):
locale.setlocale(locale.LC_ALL, '')
if clouseau.retention.utils.ignores.dir_is_ignored(dirname, self.ignored):
return todo
if clouseau.retention.utils.fileutils.dir_is_wrong_type(dirname):
return todo
if self.depth < dirname_depth:
return todo
if dirname_depth not in todo:
todo[dirname_depth] = []
if self.dirs_to_check is not None:
if clouseau.retention.utils.fileutils.subdir_check(dirname, self.dirs_to_check):
todo[dirname_depth].append(dirname)
else:
todo[dirname_depth].append(dirname)
if self.depth == dirname_depth:
# don't read below the depth level
return todo
dirs = [os.path.join(dirname, d)
for d in os.listdir(dirname)]
if self.dirs_to_check is not None:
dirs = [d for d in dirs if dirtree_check(d, self.dirs_to_check)]
for dname in dirs:
todo = self.get_subdirs_to_do(dname, dirname_depth + 1, todo)
return todo
def get_subdirs_to_do(self, dirname, dirname_depth, todo):
locale.setlocale(locale.LC_ALL, '')
if clouseau.retention.utils.ignores.dir_is_ignored(
dirname, self.ignored):
return todo
if clouseau.retention.utils.fileutils.dir_is_wrong_type(dirname):
return todo
if self.depth < dirname_depth:
return todo
if dirname_depth not in todo:
todo[dirname_depth] = []
if self.dirs_to_check is not None:
if clouseau.retention.utils.fileutils.subdir_check(
dirname, self.dirs_to_check):
todo[dirname_depth].append(dirname)
else:
todo[dirname_depth].append(dirname)
if self.depth == dirname_depth:
# don't read below the depth level
return todo
dirs = [os.path.join(dirname, d) for d in os.listdir(dirname)]
if self.dirs_to_check is not None:
dirs = [d for d in dirs if dirtree_check(d, self.dirs_to_check)]
for dname in dirs:
todo = self.get_subdirs_to_do(dname, dirname_depth + 1, todo)
return todo
def get_dirs_to_do(self, dirname):
if (self.dirs_to_check is not None and
not dirtree_check(dirname, self.dirs_to_check)):
return {}
todo = {}
depth_of_dirname = dirname.count(os.path.sep)
todo = self.get_subdirs_to_do(dirname, depth_of_dirname, todo)
return todo
def get_dirs_to_do(self, dirname):
if (self.dirs_to_check is not None
and not dirtree_check(dirname, self.dirs_to_check)):
return {}
todo = {}
depth_of_dirname = dirname.count(os.path.sep)
todo = self.get_subdirs_to_do(dirname, depth_of_dirname, todo)
return todo
def process_one_dir(self, location, subdirpath, depth, results):
'''
arguments:
location: the location being checked
subdirpath: the path to the subdirectory being checked
depth: the depth of the directory being checked (starting at 1)
results: the result set
'''
if self.dirs_to_check is not None:
if not dirtree_check(subdirpath, self.dirs_to_check):
return
if clouseau.retention.utils.ignores.dir_is_ignored(
subdirpath, self.ignored):
return True
count = 0
# doing a directory higher up in the tree than our depth cutoff,
# only do the files in it, because we have the full list of dirs
# up to our cutoff we do them one by one
if depth < self.depth:
filenames = os.listdir(subdirpath)
files = []
for fname in filenames:
try:
filestat = os.stat(os.path.join(subdirpath, fname))
except:
continue
if (not stat.S_ISLNK(filestat.st_mode)
and stat.S_ISREG(filestat.st_mode)):
files.append((fname, filestat))
self.process_files_from_path(location, subdirpath, files, count,
results)
return
# doing a directory at our cutoff depth, walk it,
# because anything below the depth
# cutoff won't be in our list
temp_results = []
for base, paths, files in self.walk_nolinks(subdirpath):
expanded_dirs, wildcard_dirs = expand_ignored_dirs(
base, self.ignored)
if self.dirs_to_check is not None:
paths[:] = [
p for p in paths
if dirtree_check(os.path.join(base, p), self.dirs_to_check)
]
paths[:] = [
p for p in paths
if (not clouseau.retention.utils.fileutils.startswithpath(
os.path.join(base, p), expanded_dirs) and
not clouseau.retention.utils.fileutils.wildcard_matches(
os.path.join(base, p), wildcard_dirs, exact=False))
]
count = self.process_files_from_path(location, base, files, count,
temp_results)
if count > self.max_files:
return
results.extend(temp_results)