def _remove_partials(self): """Remove all old .partial files. As LR now issues new downloads, and doesn't depend on these file to fix the nodes states, we just remove them to don't let garbage behind. """ try: partials = listdir(self.fsm.partials_dir) except OSError, e: if e.errno != errno.ENOENT: raise # no partials dir at all return
def test_listdir(self, expected_result=None): """Return a list of the files in a dir.""" if expected_result is None: _, valid_path_name = os.path.split(self.testfile) expected_result = [valid_path_name] for extra in ('foo', 'bar'): open_file(os.path.join(self.basedir, extra), 'w').close() expected_result.append(extra) l = listdir(self.basedir) self.assertEqual(sorted(l), sorted(expected_result)) for path in l: self.assertIsInstance(path, type(self.basedir))
def _find_data_files(self): """Collect the files we need to work with.""" logger.debug("lookingup data files") dead_files = 0 broken_files = 0 # becuase listdir appends / at the end of the path if there is not # os.path.sep at the end we are going to add it, otherwhise on windows # we will have wrong paths formed base_path = self.base_path if not base_path.endswith(os.path.sep): base_path += os.path.sep files = listdir(self.base_path) for filename in files: # first check for hint and dead if is_hint(filename): continue elif is_dead(filename): # a dead file...let's remove it dead_files += 1 DeadDataFile(self.base_path, filename).delete() continue # if it's a live or immutable file try to open it, but if it's # "broken", just rename it and continue try: if is_live(filename): self.live_file = DataFile(self.base_path, filename) elif is_immutable(filename): # it's an immutable file data_file = ImmutableDataFile(self.base_path, filename) self._immutable[data_file.file_id] = data_file except IOError as e: # oops, failed to open the file..discard it broken_files += 1 orig = os.path.join(self.base_path, filename) # get the kind of the file so we can rename it kind = LIVE if is_live(filename) else INACTIVE dest = orig.replace(kind, BROKEN) logger.warning("Failed to open %s, renaming it to: %s - " "error: %s", orig, dest, e) # rename it to "broken" rename(orig, dest) # immutable files + live logger.info( "found %s data files, %s dead and %s broken files", len(self._immutable) + 1, dead_files, broken_files )
def scan(): """The scan, really.""" log_debug("scanning the dir %r", dirpath) dircontent = listdir(dirpath) # get the info from disk dnames = [] fnames = [] for something in dircontent: fullname = os.path.join(dirpath, something) stat_result = get_stat(fullname) if stat_result is None: # gone between the listdir and now continue if is_link(fullname): log_info("Ignoring path as it's a symlink: %r", fullname) continue if not is_valid_name(fullname): m = "Ignoring path because it's invalid (non utf8): %r" log_info(m, fullname) continue if not access(fullname): log_warning("Ignoring path as we don't have enough " "permissions to track it: %r", fullname) continue if stat.S_ISDIR(stat_result.st_mode): dnames.append(something) elif stat.S_ISREG(stat_result.st_mode): fnames.append(something) else: log_warning("Path: %r isn't a dir, file or symlink.", fullname) events, to_scan_later = self._compare(dirpath, dnames, fnames, share) to_later.extend(to_scan_later) return events
def _check_move_dir(self, src, dst, real_dst): """Check that a dir was indeed moved.""" contents = sorted(listdir(src)) recursive_move(src, dst) self.assertEqual(contents, sorted(listdir(real_dst))) self.assertFalse(path_exists(src))