def _remove_partials(self):
"""Remove all old .partial files.
As LR now issues new downloads, and doesn't depend on these file to
fix the nodes states, we just remove them to don't let garbage behind.
"""
try:
partials = listdir(self.fsm.partials_dir)
except OSError, e:
if e.errno != errno.ENOENT:
raise
# no partials dir at all
return
def test_listdir(self, expected_result=None):
"""Return a list of the files in a dir."""
if expected_result is None:
_, valid_path_name = os.path.split(self.testfile)
expected_result = [valid_path_name]
for extra in ('foo', 'bar'):
open_file(os.path.join(self.basedir, extra), 'w').close()
expected_result.append(extra)
l = listdir(self.basedir)
self.assertEqual(sorted(l), sorted(expected_result))
for path in l:
self.assertIsInstance(path, type(self.basedir))
def _find_data_files(self):
"""Collect the files we need to work with."""
logger.debug("lookingup data files")
dead_files = 0
broken_files = 0
# becuase listdir appends / at the end of the path if there is not
# os.path.sep at the end we are going to add it, otherwhise on windows
# we will have wrong paths formed
base_path = self.base_path
if not base_path.endswith(os.path.sep):
base_path += os.path.sep
files = listdir(self.base_path)
for filename in files:
# first check for hint and dead
if is_hint(filename):
continue
elif is_dead(filename):
# a dead file...let's remove it
dead_files += 1
DeadDataFile(self.base_path, filename).delete()
continue
# if it's a live or immutable file try to open it, but if it's
# "broken", just rename it and continue
try:
if is_live(filename):
self.live_file = DataFile(self.base_path, filename)
elif is_immutable(filename):
# it's an immutable file
data_file = ImmutableDataFile(self.base_path, filename)
self._immutable[data_file.file_id] = data_file
except IOError as e:
# oops, failed to open the file..discard it
broken_files += 1
orig = os.path.join(self.base_path, filename)
# get the kind of the file so we can rename it
kind = LIVE if is_live(filename) else INACTIVE
dest = orig.replace(kind, BROKEN)
logger.warning("Failed to open %s, renaming it to: %s - " "error: %s", orig, dest, e)
# rename it to "broken"
rename(orig, dest)
# immutable files + live
logger.info(
"found %s data files, %s dead and %s broken files", len(self._immutable) + 1, dead_files, broken_files
)
def scan():
"""The scan, really."""
log_debug("scanning the dir %r", dirpath)
dircontent = listdir(dirpath)
# get the info from disk
dnames = []
fnames = []
for something in dircontent:
fullname = os.path.join(dirpath, something)
stat_result = get_stat(fullname)
if stat_result is None:
# gone between the listdir and now
continue
if is_link(fullname):
log_info("Ignoring path as it's a symlink: %r", fullname)
continue
if not is_valid_name(fullname):
m = "Ignoring path because it's invalid (non utf8): %r"
log_info(m, fullname)
continue
if not access(fullname):
log_warning("Ignoring path as we don't have enough "
"permissions to track it: %r", fullname)
continue
if stat.S_ISDIR(stat_result.st_mode):
dnames.append(something)
elif stat.S_ISREG(stat_result.st_mode):
fnames.append(something)
else:
log_warning("Path: %r isn't a dir, file or symlink.",
fullname)
events, to_scan_later = self._compare(dirpath, dnames, fnames,
share)
to_later.extend(to_scan_later)
return events
def _check_move_dir(self, src, dst, real_dst):
"""Check that a dir was indeed moved."""
contents = sorted(listdir(src))
recursive_move(src, dst)
self.assertEqual(contents, sorted(listdir(real_dst)))
self.assertFalse(path_exists(src))