Beispiel #1
0
def walk(targets, ftruepath=truepath, fstat=stat, fis_lnk=is_lnk, fis_dir=is_dir, funiqueid=uniqueid, fsort=sort, fdoyield=None, fskip_base=None, fskip_path=None, fskip_stats=None, depthfirst=True):
    # turn all incoming targets into TrueTarget's
    roots = {}
    for userpath in targets:
        userpath = PATH.normpath(userpath)
        abspath  = PATH.abspath(userpath)
        truepath = ftruepath(userpath)
        if truepath == None:
            continue

        # stat the path
        stat = fstat(truepath)
        if stat == None:
            continue

        # find unique id for the path
        uniqueid = funiqueid(truepath, stat)
        if uniqueid == None:
            continue

        # merge the new root target in
        if uniqueid in roots:
            roots[uniqueid].usertargets.append(UserTarget(userpath, abspath, 0))
        else:
            roots[uniqueid] = TrueTarget(truepath, stat, [UserTarget(userpath, abspath, 0)], {})

    # nothing to do?
    if len(roots) == 0:
        return
    
    # create a sorted list of roots (sorted based on truepath, and in reverse)
    roots_sorted = list(((k,),v) for k,v in roots.iteritems())
    roots_sorted.sort(key=lambda x: x[1].truepath, reverse=True)
    
    # start with nothing remaining
    remaining = deque()
    
    # Perform the tree walk
    while True:
        # Grab the next TrueTarget
        try:
            stats, uniqueids, truetarget = remaining.popleft()
        except IndexError, _:
            try:
                uniqueids, truetarget = roots_sorted.pop()
                roots.pop(uniqueids[0])
                stats = tuple(fstat(path) for path in FS.iterpath(truetarget.truepath))
            except IndexError, _:
                break
            except OSError, _:
                continue
Beispiel #2
0
def cmd_update(targets, quick=True, fset=None, skip_symlinks=False, skip_mounts=True, skip_devices=True, **kwargs):
    fskip_base = []
    fskip_path = []
    fskip_stats = []

    def fskip_base_mounts(parent, basename, userdata):
        if FS.is_mount(PATH.join(parent.truepath, basename)):
            return True
        return False
    
    def fskip_stats_rootfs(parent, path, stat, usertargets, userdata):
        if FS.is_lnk(stat):
            return False
        
        if FS.is_mount(path):
            if FS.is_sysmount(path):
                return True # skip system mounts
            userdata['root'] = path 
        elif 'root' in parent.userdata:
            userdata['root'] = parent.userdata['root']
        else:
            userdata['root'] = FS.rootpath(path)
            
        return False
    
    def fskip_stats_symlinks(parent, path, stat, usertargets, userdata):
        if FS.is_lnk(stat):
            return True
        return False
    
    def fskip_stats_devices(parent, path, stat, usertargets, userdata):
        if parent.stat.st_dev != stat.st_dev:
            return True
        return False

    if skip_mounts:
        fskip_base.append(fskip_base_mounts)

    if skip_symlinks:
        fskip_stats.append(fskip_stats_symlinks)

    if skip_devices:
        fskip_stats.append(fskip_stats_devices)
        
    fskip_stats.append(fskip_stats_rootfs)
        
    def compile(fskips):
        if len(fskips) == 0:
            return None
        def compiled(*args, **kwargs):
            return any(fskip(*args, **kwargs) for fskip in fskips)
        return compiled
    
    def fdoyield(target):
        return FS.is_reg(target.stat)
   
    # create a set of all databases
    databases = DB.Databases()
    for target in targets:
        target = FS.truepath(target)
        for path in FS.iterpath(target):
            filepath = PATH.join(path, DATABASE_BASENAME)
            if PATH.exists(filepath):
                 databases.push(FS.rootpath(path), filepath)

    # perform the walk
    for target in WALK.walk(targets, fdoyield=fdoyield, fskip_base=compile(fskip_base), fskip_path=compile(fskip_path), fskip_stats=compile(fskip_stats)):
        root = target.userdata.get('root')
        if root == None:
            root = FS.rootpath(target.truepath)
        
        if PATH.basename(target.truepath) == '.hashdb':
            databases.push(root, target.truepath)
                
        #print '[user]', target.usertargets[0].userpath
        #print root
        
        info = databases.file_get(root, target.truepath, time=target.stat.st_mtime, size=target.stat.st_size)
        data = None

        if ((info == None) and (fset == None) and (not databases.is_child(root, target.truepath))):
            print '[skip]', target.truepath
        elif (info == None)\
        or (info.hash_quick == None)\
        or ((info.hash_total == None) and (quick == False))\
        or (info.size != target.stat.st_size)\
        or (info.time != target.stat.st_mtime):
            print '[hash]', target.truepath
            filehash = HASH.hash(target.truepath, target.stat.st_size, quick)
            
            if filehash != None:
                data = {
                    'root' : root,
                    'path' : target.truepath,
                    'size' : target.stat.st_size,
                    'time' : target.stat.st_mtime,
                    'hash_quick' : filehash.hash_quick,
                    'hash_total' : filehash.hash_total
                }
        else:
            print '[keep]', target.truepath
            data = {
                'root' : root,
                'path' : info.path,
                'size' : info.size,
                'time' : info.time,
                'hash_quick' : info.hash_quick,
                'hash_total' : info.hash_total
            }
        
        if data != None:
            result = databases.file_set(**data)
            if fset != None:
                fset(target, data, result)
            
    for target in targets:
        path = FS.truepath(target)
        root = FS.rootpath(path)
        databases.purge(root, path)
    
    databases.commit()
    return databases