def check_files(test_dir, expected):
    """
    Walk test_dir.
    Check that all dirs are readable.
    Check that all files are:
     * non-special,
     * readable,
     * have a posix path that ends with one of the expected tuple paths.
    """
    result = []
    locs = []
    if filetype.is_file(test_dir):
        test_dir = fileutils.parent_directory(test_dir)

    test_dir_path = fileutils.as_posixpath(test_dir)
    for top, _, files in os.walk(test_dir):
        for f in files:
            location = os.path.join(top, f)
            locs.append(location)
            path = fileutils.as_posixpath(location)
            path = path.replace(test_dir_path, '').strip('/')
            result.append(path)

    assert sorted(expected) == sorted(result)

    for location in locs:
        assert filetype.is_file(location)
        assert not filetype.is_special(location)
        assert filetype.is_readable(location)
    def __init__(self, location):
        if (not location
            or (not os.path.exists(location)
                and not filetype.is_broken_link(location))):
            raise IOError("[Errno 2] No such file or directory: "
                          "'%(location)r'" % locals())
        self.location = location
        # flags and values
        self.is_file = filetype.is_file(location)
        self.is_dir = filetype.is_dir(location)
        self.is_regular = filetype.is_regular(location)
        self.is_special = filetype.is_special(location)

        self.date = filetype.get_last_modified_date(location)

        self.is_link = filetype.is_link(location)
        self.is_broken_link = filetype.is_broken_link(location)

        # FIXME: the way the True and False values are checked in properties is verbose and contrived at best
        # and is due to use None/True/False as different values
        # computed on demand
        self._size = None
        self._link_target = None

        self._mimetype_python = None
        self._filetype_file = None
        self._mimetype_file = None
        self._filetype_pygments = None
        self._is_pdf_with_text = None
        self._is_text = None
        self._is_binary = None
        self._contains_text = None
def check_files(test_dir, expected):
    """
    Walk test_dir.
    Check that all dirs are readable.
    Check that all files are:
     * non-special,
     * readable,
     * have a posix path that ends with one of the expected tuple paths.
    """
    result = []
    locs = []
    if filetype.is_file(test_dir):
        test_dir = fileutils.parent_directory(test_dir)

    test_dir_path = fileutils.as_posixpath(test_dir)
    for top, _, files in os.walk(test_dir):
        for f in files:
            location = os.path.join(top, f)
            locs.append(location)
            path = fileutils.as_posixpath(location)
            path = path.replace(test_dir_path, '').strip('/')
            result.append(path)

    assert sorted(expected) == sorted(result)

    for location in locs:
        assert filetype.is_file(location)
        assert not filetype.is_special(location)
        assert filetype.is_readable(location)
Пример #4
0
    def __init__(self, location):
        if (not location
            or (not os.path.exists(location)
                and not filetype.is_broken_link(location))):
            raise IOError("[Errno 2] No such file or directory: "
                          "'%(location)r'" % locals())
        self.location = location
        # flags and values
        self.is_file = filetype.is_file(location)
        self.is_dir = filetype.is_dir(location)
        self.is_regular = filetype.is_regular(location)
        self.is_special = filetype.is_special(location)

        self.date = filetype.get_last_modified_date(location)

        self.is_link = filetype.is_link(location)
        self.is_broken_link = filetype.is_broken_link(location)

        # computed on demand
        self._size = None
        self._link_target = None

        self._mimetype_python = None
        self._filetype_file = None
        self._mimetype_file = None
        self._filetype_pygments = None
        self._is_pdf_with_text = None
        self._is_text = None
        self._is_binary = None
Пример #5
0
    def __init__(self, location):
        if not location or (not os.path.exists(location) and not filetype.is_broken_link(location)):
            raise IOError("[Errno 2] No such file or directory: " "'%(location)r'" % locals())
        self.location = location
        # flags and values
        self.is_file = filetype.is_file(location)
        self.is_dir = filetype.is_dir(location)
        self.is_regular = filetype.is_regular(location)
        self.is_special = filetype.is_special(location)

        self.date = filetype.get_last_modified_date(location)

        self.is_link = filetype.is_link(location)
        self.is_broken_link = filetype.is_broken_link(location)

        # FIXME: the way the True and False values are checked in properties is verbose and contrived at best
        # and is due to use None/True/False as different values
        # computed on demand
        self._size = None
        self._link_target = None

        self._mimetype_python = None
        self._filetype_file = None
        self._mimetype_file = None
        self._filetype_pygments = None
        self._is_pdf_with_text = None
        self._is_text = None
        self._is_binary = None
Пример #6
0
def is_ignored(location, ignores, unignores=None, skip_special=True):
    """
    Return a tuple of (pattern , message) if a file at location is ignored
    or False otherwise.
    `ignores` and `unignores` are mappings of patterns to a reason.
    """
    if skip_special and filetype.is_special(location):
        return True
    return not fileset.is_included(location, includes=unignores, excludes=ignores)
Пример #7
0
def is_ignored(location, ignores, unignores, skip_special=True):
    """
    Return a tuple of (pattern , message) if a file at location is ignored
    or False otherwise.
    `ignores` and `unignores` are mappings of patterns to a reason.
    """
    if skip_special and filetype.is_special(location):
        return True
    return fileset.match(location, includes=ignores, excludes=unignores)
Пример #8
0
def walk(location, ignored=None, follow_symlinks=False):
    """
    Walk location returning the same tuples as os.walk but with a different
    behavior:
     - always walk top-down, breadth-first.
     - always ignore and never follow symlinks (unless `follow_symlinks` is True),
     - always ignore special files (FIFOs, etc.)
     - optionally ignore files and directories by invoking the `ignored`
       callable on files and directories returning True if it should be ignored.
     - location is a directory or a file: for a file, the file is returned.

    If `follow_symlinks` is True, then symlinks will not be ignored and be
    collected like regular files and directories
    """
    if on_linux and py2:
        location = fsencode(location)

    # TODO: consider using the new "scandir" module for some speed-up.

    is_ignored = ignored(location) if ignored else False
    if is_ignored:
        if TRACE:
            logger_debug('walk: ignored:', location, is_ignored)
        return

    if filetype.is_file(location, follow_symlinks=follow_symlinks):
        yield parent_directory(location), [], [file_name(location)]

    elif filetype.is_dir(location, follow_symlinks=follow_symlinks):
        dirs = []
        files = []
        # TODO: consider using scandir
        for name in os.listdir(location):
            loc = os.path.join(location, name)
            if filetype.is_special(loc) or (ignored and ignored(loc)):
                if (follow_symlinks and filetype.is_link(loc)
                        and not filetype.is_broken_link(location)):
                    pass
                else:
                    if TRACE:
                        ign = ignored and ignored(loc)
                        logger_debug('walk: ignored:', loc, ign)
                    continue
            # special files and symlinks are always ignored
            if filetype.is_dir(loc, follow_symlinks=follow_symlinks):
                dirs.append(name)
            elif filetype.is_file(loc, follow_symlinks=follow_symlinks):
                files.append(name)
        yield location, dirs, files

        for dr in dirs:
            for tripple in walk(os.path.join(location, dr),
                                ignored,
                                follow_symlinks=follow_symlinks):
                yield tripple
def check_files(test_dir, expected, regen=False):
    """
    Walk test_dir.
    Check that all dirs are readable.
    Check that all files are:
     * non-special,
     * readable,
     * have a posix path that ends with one of the expected tuple paths.
    """
    result = []
    locs = []
    if filetype.is_file(test_dir):
        test_dir = fileutils.parent_directory(test_dir)

    test_dir_path = fileutils.as_posixpath(test_dir)
    for top, _, files in os.walk(test_dir):
        for f in files:
            location = os.path.join(top, f)
            locs.append(location)
            path = fileutils.as_posixpath(location)
            path = path.replace(test_dir_path, '').strip('/')
            result.append(path)

    expected_is_json_file = False
    if not isinstance(expected, (list, tuple)) and expected.endswith('.json'):
        expected_is_json_file = True
        # this is a path to a JSON file
        if regen:
            wmode = 'wb' if py2 else 'w'
            with open(expected, wmode) as ex:
                json.dump(result, ex, indent=2, separators=(',', ':'))
            expected_content = result
        else:
            with open(expected, 'rb') as ex:
                expected_content = json.load(ex, encoding='utf-8', object_pairs_hook=OrderedDict)
    else:
        expected_content = expected

    expected_content = sorted(expected_content)
    result = sorted(result)

    try:
        assert expected_content == result
    except AssertionError:
        files = [
            'test_dir: file://{}'.format(test_dir),
            'expected: file://{}'.format(expected if expected_is_json_file else ''),
        ]
        assert files + expected_content == result

    for location in locs:
        assert filetype.is_file(location)
        assert not filetype.is_special(location)
        assert filetype.is_readable(location)
Пример #10
0
def is_ignored(location, ignores, unignores=None, skip_special=True):
    """
    Return a tuple of (pattern , message) if a file at location is ignored
    or False otherwise.
    `ignores` and `unignores` are mappings of patterns to a reason.

    If `skip_special` is True, location is checked and ignored if is considered a special file,
    e.g. symlink, FIFO, device file, etc. and location is required to be an actual location to a
    file rather than a path string.
    """
    if skip_special and filetype.is_special(location):
        return True
    return not fileset.is_included(location, includes=unignores, excludes=ignores)
Пример #11
0
def walk(location, ignored=ignore_nothing):
    """
    Walk location returning the same tuples as os.walk but with a different
    behavior:
     - always walk top-down, breadth-first.
     - always ignore and never follow symlinks, .
     - always ignore special files (FIFOs, etc.)
     - optionally ignore files and directories by invoking the `ignored`
       callable on files and directories returning True if it should be ignored.
     - location is a directory or a file: for a file, the file is returned.
    """
    if on_linux:
        location = path_to_bytes(location)

    # TODO: consider using the new "scandir" module for some speed-up.
    if TRACE:
        ign = ignored(location)
        logger_debug('walk: ignored:', location, ign)
    if ignored(location):
        return

    if filetype.is_file(location):
        yield parent_directory(location), [], [file_name(location)]

    elif filetype.is_dir(location):
        dirs = []
        files = []
        # TODO: consider using scandir
        for name in os.listdir(location):
            loc = os.path.join(location, name)
            if filetype.is_special(loc) or ignored(loc):
                if TRACE:
                    ign = ignored(loc)
                    logger_debug('walk: ignored:', loc, ign)
                continue
            # special files and symlinks are always ignored
            if filetype.is_dir(loc):
                dirs.append(name)
            elif filetype.is_file(loc):
                files.append(name)
        yield location, dirs, files

        for dr in dirs:
            for tripple in walk(os.path.join(location, dr), ignored):
                yield tripple
Пример #12
0
def walk(location, ignored=ignore_nothing):
    """
    Walk location returning the same tuples as os.walk but with a different
    behavior:
     - always walk top-down, breadth-first.
     - always ignore and never follow symlinks, .
     - always ignore special files (FIFOs, etc.)
     - optionally ignore files and directories by invoking the `ignored`
       callable on files and directories returning True if it should be ignored.
     - location is a directory or a file: for a file, the file is returned.
    """
    if on_linux:
        location = path_to_bytes(location)

    # TODO: consider using the new "scandir" module for some speed-up.
    if TRACE:
        ign = ignored(location)
        logger_debug('walk: ignored:', location, ign)
    if ignored(location):
        return

    if filetype.is_file(location) :
        yield parent_directory(location), [], [file_name(location)]

    elif filetype.is_dir(location):
        dirs = []
        files = []
        # TODO: consider using scandir
        for name in os.listdir(location):
            loc = os.path.join(location, name)
            if filetype.is_special(loc) or ignored(loc):
                if TRACE:
                    ign = ignored(loc)
                    logger_debug('walk: ignored:', loc, ign)
                continue
            # special files and symlinks are always ignored
            if filetype.is_dir(loc):
                dirs.append(name)
            elif filetype.is_file(loc):
                files.append(name)
        yield location, dirs, files

        for dr in dirs:
            for tripple in walk(os.path.join(location, dr), ignored):
                yield tripple