コード例 #1
0
ファイル: hashdb_walk.py プロジェクト: Douglozy/hashdb
    def walk(self):
        '''Walks the directory tree specified by targets, yielding all accessible regular files as Walker.Target objects'''

        # compile all the information required to walk the targets
        targets = set(self._targets)
        fskip_fstype = self.build_fskip_globs(self._skip_fstypes)
        fskip_path = self.build_fskip_globs(self._skip_paths)
        fskip_name = self.build_fskip_globs(self._skip_names)
        fskip_dirname = self.build_fskip_globs(self._skip_dirnames)
        fskip_filename = self.build_fskip_globs(self._skip_filenames)
        fskip_access = None
        skip_binds = self._skip_binds
        skip_mounts = self._skip_mounts
        skip_symlinks = self._skip_symlinks
        is_linuxy = False
        mounts = MountEntries()

        if platform.system() != 'Windows':
            is_linuxy = True

            def fskip_access(target):
                access = stat.S_IMODE(target.stat.st_mode)
                if access & (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) == 0:
                    # usr, grp, oth : no access
                    return True
                elif target.stat.st_uid == euid:
                    # usr
                    if access & stat.S_IRUSR == 0:
                        return True
                elif access & (stat.S_IRGRP | stat.S_IROTH) == 0:
                    # grp, oth : no access
                    return True
                elif target.stat.st_gid in groups:
                    # grp
                    if access & stat.S_IRGRP == 0:
                        return True
                elif access & (stat.S_IROTH) == 0:
                    # oth : no access
                    return True

                # Note that this may return some erronious negatives.
                # The objective is to try to avoid access errors, not prevent them entirely
                # Note that we could also use os.access, but that would cause additional
                # os.stat calls, and we want speed
                return False

            fskip_access.func_globals['uid'] = os.getuid()
            fskip_access.func_globals['euid'] = os.geteuid()
            fskip_access.func_globals['gid'] = os.getgid()
            fskip_access.func_globals['egid'] = os.getegid()
            fskip_access.func_globals['groups'] = os.getgroups()

            if os.geteuid() == 0:
                fskip_access = None  # No need to skip things if we are root

        try:
            todo = deque()
            dirs = deque()

            if self.walk_depth:
                fappend = dirs.appendleft

                def fdone():
                    todo.extendleft(dirs)
                    dirs.clear()
            else:
                fappend = todo.append

                def fdone():
                    pass

            for target in targets:
                log.verbose(PREFIX_ROOT + '%s (root)' % target.user)

                if stat.S_ISREG(target.stat.st_mode):
                    yield target
                if stat.S_ISDIR(target.stat.st_mode):
                    todo.clear()
                    todo.append((target, [(target.stat.st_ino,
                                           target.stat.st_dev)]))
                    while True:
                        try:
                            target, nodes = todo.popleft()
                        except IndexError, _:
                            break  # Reached the last element in the list

                        try:
                            filelist = os.listdir(target.true)
                            filelist.sort()
                        except OSError, ex:
                            log.warning('warning: Unable to list target %r: %s'
                                        % (target.user, ex))
                            continue

                        for name in filelist:
                            child = Walker.Target(
                                os.path.join(target.true, name),
                                os.path.join(target.user, name), None)

                            # skip name?
                            if fskip_name and fskip_name(name):
                                if log.is_debug:
                                    log.debug(PREFIX_SKIP +
                                              '%s (skip_name)' % child.user)
                                continue

                            # skip path?
                            if fskip_path and fskip_path(child.user):
                                if log.is_debug:
                                    log.debug(PREFIX_SKIP +
                                              '%s (skip_path)' % child.user)
                                continue

                            # stat
                            try:
                                child = child._replace(
                                    stat=os.lstat(child.true))
                            except OSError, ex:
                                log.warning('warning: Unable to lstat %r: %s' %
                                            (child.user, ex))
                                if log.is_debug:
                                    log.debug(PREFIX_SKIP +
                                              '%r (failed lstat)' % child.user)
                                continue

                            # recursive loop?
                            if (child.stat.st_ino, child.stat.st_dev) in nodes:
                                log.debug(
                                    PREFIX_SKIP +
                                    '%r (loop chain detected)' % child.user)
                                continue

                            # check access
                            if fskip_access and fskip_access(child):
                                log.debug(PREFIX_SKIP +
                                          '%r (no access)' % child.user)
                                continue

                            # resolve symlinks
                            if stat.S_ISLNK(child.stat.st_mode):
                                if skip_symlinks:
                                    log.debug(
                                        PREFIX_SKIP +
                                        '%r (skip_symlinks)' % child.user)
                                    continue

                                log.debug(PREFIX_SYM +
                                          '%s (sym link)' % child.user)

                                try:
                                    child = child._replace(
                                        true=mounts.truepath(child.true))
                                    child = child._replace(
                                        stat=os.lstat(child.true))
                                except OSError, ex:
                                    log.warning(
                                        'warning: Unable to read symlink target %r: %s'
                                        % (child.user, ex))
                                    if log.is_debug:
                                        log.debug(
                                            PREFIX_SKIP +
                                            '%r (failed to read symlink target)'
                                            % child.user)
                                    continue

                                # recursive loop?
                                if (child.stat.st_ino,
                                        child.stat.st_dev) in nodes:
                                    log.debug(PREFIX_SKIP +
                                              '%r (loop chain detected)' %
                                              child.user)
                                    continue

                                # check access
                                if fskip_access != None and fskip_access(
                                        child):
                                    log.debug(PREFIX_SKIP +
                                              '%r (no access)' % child.user)
                                    continue

                                # Need to recalculate child.true/..
                                parent_stat = None
                            else:
                                parent_stat = target.stat

                            # regular file?
                            if stat.S_ISREG(child.stat.st_mode):
                                # skip filename?
                                if fskip_filename and fskip_filename(name):
                                    if log.is_debug:
                                        log.debug(
                                            PREFIX_SKIP +
                                            '%r (skip_filename)' % child.user)
                                    continue

                                if log.is_verbose:
                                    log.verbose(
                                        PREFIX_REG +
                                        '%s (regular file)' % child.user)

                                yield child
                                continue

                            # directory?
                            if stat.S_ISDIR(child.stat.st_mode):
                                # skip dirname?
                                if fskip_dirname and fskip_dirname(name):
                                    if log.is_debug:
                                        log.debug(
                                            PREFIX_SKIP +
                                            '%s (skip_dirname)' % child.user)
                                    continue

                                # is bind? ToDo: Should this check be in a loop for bind chains?
                                if mounts.is_bind(child.true):
                                    # skip binds?
                                    if skip_binds:
                                        if log.is_debug:
                                            log.debug(
                                                PREFIX_SKIP +
                                                '%s (skip_binds)' % child.user)
                                        continue

                                    log.debug(PREFIX_BIND +
                                              '%s (bind mount)' % child.user)

                                    try:
                                        child = child._replace(
                                            true=mounts.truepath(child.true))
                                        child = child._replace(
                                            stat=os.lstat(child.true))
                                    except OSError, ex:
                                        log.warning(
                                            'warning: Unable to read bind target %r: %s'
                                            % (child.user, ex))
                                        if log.is_debug:
                                            log.debug(
                                                PREFIX_SKIP +
                                                '%r (failed to read bind target)'
                                                % (child.user, ex))
                                        continue

                                    # recursive loop?
                                    if (child.stat.st_ino,
                                            child.stat.st_dev) in nodes:
                                        log.debug(PREFIX_SKIP +
                                                  '%r (loop chain detected)' %
                                                  child.user)
                                        continue

                                    # check access
                                    if fskip_access != None and fskip_access(
                                            child):
                                        log.debug(
                                            PREFIX_SKIP +
                                            '%r (no access)' % child.user)
                                        continue

                                    parent_stat = None

                                # get parent stat
                                if is_linuxy and parent_stat == None:
                                    try:
                                        parent_stat = os.lstat(
                                            os.path.join(child.true, '..'))
                                    except OSError, ex:
                                        log.warning(
                                            'warning: Unable to read parent %r: %s'
                                            % (os.path.join(child.user, '..'),
                                               ex))

                                # is mount?
                                keep = None
                                if  (is_linuxy and (parent_stat != None))\
                                and ((parent_stat.st_dev != child.stat.st_dev)\
                                or   (parent_stat.st_ino == child.stat.st_ino)):
                                    # skip mounts?
                                    if skip_mounts:
                                        log.debug(
                                            PREFIX_SKIP +
                                            '%r (skip_mounts)' % child.user)
                                        continue

                                    # skip fstype?
                                    if fskip_fstype:
                                        # find fstype, updating mount points if required
                                        fstype = mounts.get_fstype(child.true)
                                        ##if fstype == None:
                                        ##    mounts = MountEntries()
                                        ##    fstype = mounts.get_fstype(child.true)
                                        if fstype == None:
                                            log.warning(
                                                'warning: Unable to resolve mount fstype %r'
                                                % child.user)
                                            log.debug(
                                                PREFIX_SKIP +
                                                '%s (failed to resolve mount fstype)'
                                                % child.user)
                                            continue
                                        mounts = mounts

                                        if fskip_fstype(fstype.type):
                                            if log.is_debug:
                                                log.debug(PREFIX_SKIP +
                                                          '%s (skip_fstype)' %
                                                          child.user)
                                            continue

                                # directory
                                if log.is_verbose:
                                    log.verbose(PREFIX_DIR +
                                                '%s (directory)' % child.user)

                                # put directory in the todo list
                                fappend((child, nodes + [(child.stat.st_ino,
                                                          child.stat.st_dev)]))
                                continue
コード例 #2
0
ファイル: hashdb.py プロジェクト: Douglozy/hashdb
    def view(self):
        # Setup logging
        log.setLevel(self.setting_verbosity)

        # Display configuration
        display_settings(self.settings, log.debug)

        # Setup database
        log.debug("* setup database's...")
        db = HashDatabase(self.setting_database)
        db.add_combines(self.setting_combine)
        if not db.open():
            return

        # Read mounts (for truepath)
        mounts = MountEntries()

        # Build a query string, filtering on targets
        targets = [mounts.truepath(t) for t in self.setting_targets]
        qfilters = []
        qargmap = {}
        if ("/" not in targets) and ("\\" not in targets) and ("//" not in targets) and ("\\\\" not in targets):
            for i, target in enumerate(targets):
                target = mounts.truepath(target)
                qfilters.append(
                    r"""(path = :%(name)s) OR (substr(path, 1, :%(name)s_len + 1) = :%(name)s || '/')"""
                    % {"name": "t%02d" % i}
                )
                qargmap.update({"t%02d" % i: target, "t%02d_len" % i: len(target)})
        qfilter = (r"""WHERE """ + r""" OR """.join(qfilters)) if len(qfilters) != 0 else r""""""
        qorder = (
            r"""
            ORDER BY
                path,
                mark DESC
        """
            if self.setting_walk_depth
            else r"""
            ORDER BY
                count_components(path),
                path,
                mark DESC
        """
        )

        query = (
            r"""
            SELECT
                *
            FROM
                combinedtab
        """
            + qfilter
            + qorder
        )

        # yield all results as a HashRowData blob (don't expose the underlying row)
        for row in db.connection.execute(query, qargmap):
            yield (
                HashRowData(path=row["path"], hash=row["hash"], mark=row["mark"], time=row["time"], size=row["size"]),
                db,
            )