def check_files(test_dir, expected): """ Walk test_dir. Check that all dirs are readable. Check that all files are: * non-special, * readable, * have a posix path that ends with one of the expected tuple paths. """ result = [] locs = [] if filetype.is_file(test_dir): test_dir = fileutils.parent_directory(test_dir) test_dir_path = fileutils.as_posixpath(test_dir) for top, _, files in os.walk(test_dir): for f in files: location = os.path.join(top, f) locs.append(location) path = fileutils.as_posixpath(location) path = path.replace(test_dir_path, '').strip('/') result.append(path) assert sorted(expected) == sorted(result) for location in locs: assert filetype.is_file(location) assert not filetype.is_special(location) assert filetype.is_readable(location)
def __init__(self, location): if (not location or (not os.path.exists(location) and not filetype.is_broken_link(location))): raise IOError("[Errno 2] No such file or directory: " "'%(location)r'" % locals()) self.location = location # flags and values self.is_file = filetype.is_file(location) self.is_dir = filetype.is_dir(location) self.is_regular = filetype.is_regular(location) self.is_special = filetype.is_special(location) self.date = filetype.get_last_modified_date(location) self.is_link = filetype.is_link(location) self.is_broken_link = filetype.is_broken_link(location) # FIXME: the way the True and False values are checked in properties is verbose and contrived at best # and is due to use None/True/False as different values # computed on demand self._size = None self._link_target = None self._mimetype_python = None self._filetype_file = None self._mimetype_file = None self._filetype_pygments = None self._is_pdf_with_text = None self._is_text = None self._is_binary = None self._contains_text = None
def __init__(self, location): if (not location or (not os.path.exists(location) and not filetype.is_broken_link(location))): raise IOError("[Errno 2] No such file or directory: " "'%(location)r'" % locals()) self.location = location # flags and values self.is_file = filetype.is_file(location) self.is_dir = filetype.is_dir(location) self.is_regular = filetype.is_regular(location) self.is_special = filetype.is_special(location) self.date = filetype.get_last_modified_date(location) self.is_link = filetype.is_link(location) self.is_broken_link = filetype.is_broken_link(location) # computed on demand self._size = None self._link_target = None self._mimetype_python = None self._filetype_file = None self._mimetype_file = None self._filetype_pygments = None self._is_pdf_with_text = None self._is_text = None self._is_binary = None
def __init__(self, location): if not location or (not os.path.exists(location) and not filetype.is_broken_link(location)): raise IOError("[Errno 2] No such file or directory: " "'%(location)r'" % locals()) self.location = location # flags and values self.is_file = filetype.is_file(location) self.is_dir = filetype.is_dir(location) self.is_regular = filetype.is_regular(location) self.is_special = filetype.is_special(location) self.date = filetype.get_last_modified_date(location) self.is_link = filetype.is_link(location) self.is_broken_link = filetype.is_broken_link(location) # FIXME: the way the True and False values are checked in properties is verbose and contrived at best # and is due to use None/True/False as different values # computed on demand self._size = None self._link_target = None self._mimetype_python = None self._filetype_file = None self._mimetype_file = None self._filetype_pygments = None self._is_pdf_with_text = None self._is_text = None self._is_binary = None
def is_ignored(location, ignores, unignores=None, skip_special=True): """ Return a tuple of (pattern , message) if a file at location is ignored or False otherwise. `ignores` and `unignores` are mappings of patterns to a reason. """ if skip_special and filetype.is_special(location): return True return not fileset.is_included(location, includes=unignores, excludes=ignores)
def is_ignored(location, ignores, unignores, skip_special=True): """ Return a tuple of (pattern , message) if a file at location is ignored or False otherwise. `ignores` and `unignores` are mappings of patterns to a reason. """ if skip_special and filetype.is_special(location): return True return fileset.match(location, includes=ignores, excludes=unignores)
def walk(location, ignored=None, follow_symlinks=False): """ Walk location returning the same tuples as os.walk but with a different behavior: - always walk top-down, breadth-first. - always ignore and never follow symlinks (unless `follow_symlinks` is True), - always ignore special files (FIFOs, etc.) - optionally ignore files and directories by invoking the `ignored` callable on files and directories returning True if it should be ignored. - location is a directory or a file: for a file, the file is returned. If `follow_symlinks` is True, then symlinks will not be ignored and be collected like regular files and directories """ if on_linux and py2: location = fsencode(location) # TODO: consider using the new "scandir" module for some speed-up. is_ignored = ignored(location) if ignored else False if is_ignored: if TRACE: logger_debug('walk: ignored:', location, is_ignored) return if filetype.is_file(location, follow_symlinks=follow_symlinks): yield parent_directory(location), [], [file_name(location)] elif filetype.is_dir(location, follow_symlinks=follow_symlinks): dirs = [] files = [] # TODO: consider using scandir for name in os.listdir(location): loc = os.path.join(location, name) if filetype.is_special(loc) or (ignored and ignored(loc)): if (follow_symlinks and filetype.is_link(loc) and not filetype.is_broken_link(location)): pass else: if TRACE: ign = ignored and ignored(loc) logger_debug('walk: ignored:', loc, ign) continue # special files and symlinks are always ignored if filetype.is_dir(loc, follow_symlinks=follow_symlinks): dirs.append(name) elif filetype.is_file(loc, follow_symlinks=follow_symlinks): files.append(name) yield location, dirs, files for dr in dirs: for tripple in walk(os.path.join(location, dr), ignored, follow_symlinks=follow_symlinks): yield tripple
def check_files(test_dir, expected, regen=False): """ Walk test_dir. Check that all dirs are readable. Check that all files are: * non-special, * readable, * have a posix path that ends with one of the expected tuple paths. """ result = [] locs = [] if filetype.is_file(test_dir): test_dir = fileutils.parent_directory(test_dir) test_dir_path = fileutils.as_posixpath(test_dir) for top, _, files in os.walk(test_dir): for f in files: location = os.path.join(top, f) locs.append(location) path = fileutils.as_posixpath(location) path = path.replace(test_dir_path, '').strip('/') result.append(path) expected_is_json_file = False if not isinstance(expected, (list, tuple)) and expected.endswith('.json'): expected_is_json_file = True # this is a path to a JSON file if regen: wmode = 'wb' if py2 else 'w' with open(expected, wmode) as ex: json.dump(result, ex, indent=2, separators=(',', ':')) expected_content = result else: with open(expected, 'rb') as ex: expected_content = json.load(ex, encoding='utf-8', object_pairs_hook=OrderedDict) else: expected_content = expected expected_content = sorted(expected_content) result = sorted(result) try: assert expected_content == result except AssertionError: files = [ 'test_dir: file://{}'.format(test_dir), 'expected: file://{}'.format(expected if expected_is_json_file else ''), ] assert files + expected_content == result for location in locs: assert filetype.is_file(location) assert not filetype.is_special(location) assert filetype.is_readable(location)
def is_ignored(location, ignores, unignores=None, skip_special=True): """ Return a tuple of (pattern , message) if a file at location is ignored or False otherwise. `ignores` and `unignores` are mappings of patterns to a reason. If `skip_special` is True, location is checked and ignored if is considered a special file, e.g. symlink, FIFO, device file, etc. and location is required to be an actual location to a file rather than a path string. """ if skip_special and filetype.is_special(location): return True return not fileset.is_included(location, includes=unignores, excludes=ignores)
def walk(location, ignored=ignore_nothing): """ Walk location returning the same tuples as os.walk but with a different behavior: - always walk top-down, breadth-first. - always ignore and never follow symlinks, . - always ignore special files (FIFOs, etc.) - optionally ignore files and directories by invoking the `ignored` callable on files and directories returning True if it should be ignored. - location is a directory or a file: for a file, the file is returned. """ if on_linux: location = path_to_bytes(location) # TODO: consider using the new "scandir" module for some speed-up. if TRACE: ign = ignored(location) logger_debug('walk: ignored:', location, ign) if ignored(location): return if filetype.is_file(location): yield parent_directory(location), [], [file_name(location)] elif filetype.is_dir(location): dirs = [] files = [] # TODO: consider using scandir for name in os.listdir(location): loc = os.path.join(location, name) if filetype.is_special(loc) or ignored(loc): if TRACE: ign = ignored(loc) logger_debug('walk: ignored:', loc, ign) continue # special files and symlinks are always ignored if filetype.is_dir(loc): dirs.append(name) elif filetype.is_file(loc): files.append(name) yield location, dirs, files for dr in dirs: for tripple in walk(os.path.join(location, dr), ignored): yield tripple
def walk(location, ignored=ignore_nothing): """ Walk location returning the same tuples as os.walk but with a different behavior: - always walk top-down, breadth-first. - always ignore and never follow symlinks, . - always ignore special files (FIFOs, etc.) - optionally ignore files and directories by invoking the `ignored` callable on files and directories returning True if it should be ignored. - location is a directory or a file: for a file, the file is returned. """ if on_linux: location = path_to_bytes(location) # TODO: consider using the new "scandir" module for some speed-up. if TRACE: ign = ignored(location) logger_debug('walk: ignored:', location, ign) if ignored(location): return if filetype.is_file(location) : yield parent_directory(location), [], [file_name(location)] elif filetype.is_dir(location): dirs = [] files = [] # TODO: consider using scandir for name in os.listdir(location): loc = os.path.join(location, name) if filetype.is_special(loc) or ignored(loc): if TRACE: ign = ignored(loc) logger_debug('walk: ignored:', loc, ign) continue # special files and symlinks are always ignored if filetype.is_dir(loc): dirs.append(name) elif filetype.is_file(loc): files.append(name) yield location, dirs, files for dr in dirs: for tripple in walk(os.path.join(location, dr), ignored): yield tripple