def check_files(test_dir, expected):
"""
Walk test_dir.
Check that all dirs are readable.
Check that all files are:
* non-special,
* readable,
* have a posix path that ends with one of the expected tuple paths.
"""
result = []
locs = []
if filetype.is_file(test_dir):
test_dir = fileutils.parent_directory(test_dir)
test_dir_path = fileutils.as_posixpath(test_dir)
for top, _, files in os.walk(test_dir):
for f in files:
location = os.path.join(top, f)
locs.append(location)
path = fileutils.as_posixpath(location)
path = path.replace(test_dir_path, '').strip('/')
result.append(path)
assert sorted(expected) == sorted(result)
for location in locs:
assert filetype.is_file(location)
assert not filetype.is_special(location)
assert filetype.is_readable(location)
def __init__(self, location):
if (not location
or (not os.path.exists(location)
and not filetype.is_broken_link(location))):
raise IOError("[Errno 2] No such file or directory: "
"'%(location)r'" % locals())
self.location = location
# flags and values
self.is_file = filetype.is_file(location)
self.is_dir = filetype.is_dir(location)
self.is_regular = filetype.is_regular(location)
self.is_special = filetype.is_special(location)
self.date = filetype.get_last_modified_date(location)
self.is_link = filetype.is_link(location)
self.is_broken_link = filetype.is_broken_link(location)
# FIXME: the way the True and False values are checked in properties is verbose and contrived at best
# and is due to use None/True/False as different values
# computed on demand
self._size = None
self._link_target = None
self._mimetype_python = None
self._filetype_file = None
self._mimetype_file = None
self._filetype_pygments = None
self._is_pdf_with_text = None
self._is_text = None
self._is_binary = None
self._contains_text = None
def check_files(test_dir, expected):
"""
Walk test_dir.
Check that all dirs are readable.
Check that all files are:
* non-special,
* readable,
* have a posix path that ends with one of the expected tuple paths.
"""
result = []
locs = []
if filetype.is_file(test_dir):
test_dir = fileutils.parent_directory(test_dir)
test_dir_path = fileutils.as_posixpath(test_dir)
for top, _, files in os.walk(test_dir):
for f in files:
location = os.path.join(top, f)
locs.append(location)
path = fileutils.as_posixpath(location)
path = path.replace(test_dir_path, '').strip('/')
result.append(path)
assert sorted(expected) == sorted(result)
for location in locs:
assert filetype.is_file(location)
assert not filetype.is_special(location)
assert filetype.is_readable(location)
def __init__(self, location):
if (not location
or (not os.path.exists(location)
and not filetype.is_broken_link(location))):
raise IOError("[Errno 2] No such file or directory: "
"'%(location)r'" % locals())
self.location = location
# flags and values
self.is_file = filetype.is_file(location)
self.is_dir = filetype.is_dir(location)
self.is_regular = filetype.is_regular(location)
self.is_special = filetype.is_special(location)
self.date = filetype.get_last_modified_date(location)
self.is_link = filetype.is_link(location)
self.is_broken_link = filetype.is_broken_link(location)
# computed on demand
self._size = None
self._link_target = None
self._mimetype_python = None
self._filetype_file = None
self._mimetype_file = None
self._filetype_pygments = None
self._is_pdf_with_text = None
self._is_text = None
self._is_binary = None
def __init__(self, location):
if not location or (not os.path.exists(location) and not filetype.is_broken_link(location)):
raise IOError("[Errno 2] No such file or directory: " "'%(location)r'" % locals())
self.location = location
# flags and values
self.is_file = filetype.is_file(location)
self.is_dir = filetype.is_dir(location)
self.is_regular = filetype.is_regular(location)
self.is_special = filetype.is_special(location)
self.date = filetype.get_last_modified_date(location)
self.is_link = filetype.is_link(location)
self.is_broken_link = filetype.is_broken_link(location)
# FIXME: the way the True and False values are checked in properties is verbose and contrived at best
# and is due to use None/True/False as different values
# computed on demand
self._size = None
self._link_target = None
self._mimetype_python = None
self._filetype_file = None
self._mimetype_file = None
self._filetype_pygments = None
self._is_pdf_with_text = None
self._is_text = None
self._is_binary = None
def is_ignored(location, ignores, unignores=None, skip_special=True):
"""
Return a tuple of (pattern , message) if a file at location is ignored
or False otherwise.
`ignores` and `unignores` are mappings of patterns to a reason.
"""
if skip_special and filetype.is_special(location):
return True
return not fileset.is_included(location, includes=unignores, excludes=ignores)
def is_ignored(location, ignores, unignores, skip_special=True):
"""
Return a tuple of (pattern , message) if a file at location is ignored
or False otherwise.
`ignores` and `unignores` are mappings of patterns to a reason.
"""
if skip_special and filetype.is_special(location):
return True
return fileset.match(location, includes=ignores, excludes=unignores)
def walk(location, ignored=None, follow_symlinks=False):
"""
Walk location returning the same tuples as os.walk but with a different
behavior:
- always walk top-down, breadth-first.
- always ignore and never follow symlinks (unless `follow_symlinks` is True),
- always ignore special files (FIFOs, etc.)
- optionally ignore files and directories by invoking the `ignored`
callable on files and directories returning True if it should be ignored.
- location is a directory or a file: for a file, the file is returned.
If `follow_symlinks` is True, then symlinks will not be ignored and be
collected like regular files and directories
"""
if on_linux and py2:
location = fsencode(location)
# TODO: consider using the new "scandir" module for some speed-up.
is_ignored = ignored(location) if ignored else False
if is_ignored:
if TRACE:
logger_debug('walk: ignored:', location, is_ignored)
return
if filetype.is_file(location, follow_symlinks=follow_symlinks):
yield parent_directory(location), [], [file_name(location)]
elif filetype.is_dir(location, follow_symlinks=follow_symlinks):
dirs = []
files = []
# TODO: consider using scandir
for name in os.listdir(location):
loc = os.path.join(location, name)
if filetype.is_special(loc) or (ignored and ignored(loc)):
if (follow_symlinks and filetype.is_link(loc)
and not filetype.is_broken_link(location)):
pass
else:
if TRACE:
ign = ignored and ignored(loc)
logger_debug('walk: ignored:', loc, ign)
continue
# special files and symlinks are always ignored
if filetype.is_dir(loc, follow_symlinks=follow_symlinks):
dirs.append(name)
elif filetype.is_file(loc, follow_symlinks=follow_symlinks):
files.append(name)
yield location, dirs, files
for dr in dirs:
for tripple in walk(os.path.join(location, dr),
ignored,
follow_symlinks=follow_symlinks):
yield tripple
def check_files(test_dir, expected, regen=False):
"""
Walk test_dir.
Check that all dirs are readable.
Check that all files are:
* non-special,
* readable,
* have a posix path that ends with one of the expected tuple paths.
"""
result = []
locs = []
if filetype.is_file(test_dir):
test_dir = fileutils.parent_directory(test_dir)
test_dir_path = fileutils.as_posixpath(test_dir)
for top, _, files in os.walk(test_dir):
for f in files:
location = os.path.join(top, f)
locs.append(location)
path = fileutils.as_posixpath(location)
path = path.replace(test_dir_path, '').strip('/')
result.append(path)
expected_is_json_file = False
if not isinstance(expected, (list, tuple)) and expected.endswith('.json'):
expected_is_json_file = True
# this is a path to a JSON file
if regen:
wmode = 'wb' if py2 else 'w'
with open(expected, wmode) as ex:
json.dump(result, ex, indent=2, separators=(',', ':'))
expected_content = result
else:
with open(expected, 'rb') as ex:
expected_content = json.load(ex, encoding='utf-8', object_pairs_hook=OrderedDict)
else:
expected_content = expected
expected_content = sorted(expected_content)
result = sorted(result)
try:
assert expected_content == result
except AssertionError:
files = [
'test_dir: file://{}'.format(test_dir),
'expected: file://{}'.format(expected if expected_is_json_file else ''),
]
assert files + expected_content == result
for location in locs:
assert filetype.is_file(location)
assert not filetype.is_special(location)
assert filetype.is_readable(location)
def is_ignored(location, ignores, unignores=None, skip_special=True):
"""
Return a tuple of (pattern , message) if a file at location is ignored
or False otherwise.
`ignores` and `unignores` are mappings of patterns to a reason.
If `skip_special` is True, location is checked and ignored if is considered a special file,
e.g. symlink, FIFO, device file, etc. and location is required to be an actual location to a
file rather than a path string.
"""
if skip_special and filetype.is_special(location):
return True
return not fileset.is_included(location, includes=unignores, excludes=ignores)
def walk(location, ignored=ignore_nothing):
"""
Walk location returning the same tuples as os.walk but with a different
behavior:
- always walk top-down, breadth-first.
- always ignore and never follow symlinks, .
- always ignore special files (FIFOs, etc.)
- optionally ignore files and directories by invoking the `ignored`
callable on files and directories returning True if it should be ignored.
- location is a directory or a file: for a file, the file is returned.
"""
if on_linux:
location = path_to_bytes(location)
# TODO: consider using the new "scandir" module for some speed-up.
if TRACE:
ign = ignored(location)
logger_debug('walk: ignored:', location, ign)
if ignored(location):
return
if filetype.is_file(location):
yield parent_directory(location), [], [file_name(location)]
elif filetype.is_dir(location):
dirs = []
files = []
# TODO: consider using scandir
for name in os.listdir(location):
loc = os.path.join(location, name)
if filetype.is_special(loc) or ignored(loc):
if TRACE:
ign = ignored(loc)
logger_debug('walk: ignored:', loc, ign)
continue
# special files and symlinks are always ignored
if filetype.is_dir(loc):
dirs.append(name)
elif filetype.is_file(loc):
files.append(name)
yield location, dirs, files
for dr in dirs:
for tripple in walk(os.path.join(location, dr), ignored):
yield tripple
def walk(location, ignored=ignore_nothing):
"""
Walk location returning the same tuples as os.walk but with a different
behavior:
- always walk top-down, breadth-first.
- always ignore and never follow symlinks, .
- always ignore special files (FIFOs, etc.)
- optionally ignore files and directories by invoking the `ignored`
callable on files and directories returning True if it should be ignored.
- location is a directory or a file: for a file, the file is returned.
"""
if on_linux:
location = path_to_bytes(location)
# TODO: consider using the new "scandir" module for some speed-up.
if TRACE:
ign = ignored(location)
logger_debug('walk: ignored:', location, ign)
if ignored(location):
return
if filetype.is_file(location) :
yield parent_directory(location), [], [file_name(location)]
elif filetype.is_dir(location):
dirs = []
files = []
# TODO: consider using scandir
for name in os.listdir(location):
loc = os.path.join(location, name)
if filetype.is_special(loc) or ignored(loc):
if TRACE:
ign = ignored(loc)
logger_debug('walk: ignored:', loc, ign)
continue
# special files and symlinks are always ignored
if filetype.is_dir(loc):
dirs.append(name)
elif filetype.is_file(loc):
files.append(name)
yield location, dirs, files
for dr in dirs:
for tripple in walk(os.path.join(location, dr), ignored):
yield tripple