예제 #1
0
파일: Channels.py 프로젝트: pasalo/common
    def get_local_files (self, channels):
        downloads_dir = self.config.config['downloads']

        files = []
        for channel in channels:
            if '..' in channel:
                continue

            dir_fp = os.path.join (downloads_dir, channel)

            if not os.path.exists (dir_fp):
                continue
            if not os.path.isdir (dir_fp):
                continue

            for f in os.listdir (dir_fp):
                if f[0] in '.#~':
                    continue

                fname      = os.path.join (channel, f)
                fname_fp   = os.path.join (dir_fp, f)
                fname_size = os.path.getsize(fname_fp)

                # No dirs within channels
                if not os.path.isfile (fname_fp):
                    continue

                utils.set_md5_attr (fname_fp, force=False)
                fname_md5 = xattr.getxattr (fname_fp, 'md5')

                files.append ({'path': fname, 'size': fname_size, 'md5': fname_md5})

        return files
예제 #2
0
def getxattr(fullpath, attr, default=None):
    try:
        return xattr.getxattr(fullpath, attr)
    except IOError, e:
        # Attribute not found
        if e.errno == 93:
            return default
        raise
예제 #3
0
파일: utils.py 프로젝트: pasalo/common
def getxattr (fullpath, attr, default=None):
    try:
        return xattr.getxattr (fullpath, attr)
    except IOError, e:
        # Attribute not found
        if e.errno == 93:
            return default
        raise
예제 #4
0
    def execute (self):
        # Channels
        lst = ChannelList (self.config, self.keys, self.id)
        channels = lst.execute()

        # Files
        lst = FileList (self.config, self.keys, self.id, ','.join(channels))
        remote_files = lst.execute()

        # Check local files
        new_files = []

        for remote_file in remote_files:
            fp = os.path.join (self.download_dir, remote_file['path'])

            # No local version
            if not os.path.exists (fp):
                new_files.append (remote_file)
                continue

            # Outdated local version
            size      = os.path.getsize(fp)
            attr_md5  = xattr.getxattr (fp, 'md5')
            attr_time = utils.getxattr (fp, 'md5_time', 0)

            if remote_file['size'] != size:
                new_files.append (remote_file)
                continue

            if remote_file['md5']  != attr_md5:
                new_files.append (remote_file)
                continue

            logging.info ("%s is up to date" %(remote_file['path']))

        if not new_files:
            return

        # Report
        for f in new_files:
            channel, filename = f['path'].split('/', 1)
            print ('  #%s - %s (%s)' %(channel, filename, utils.format_size(f['size'])))

        total_size = reduce (lambda x,y: x+y, [f['size'] for f in new_files])
        print ("%d files: %s"%(len(new_files), utils.format_size(total_size)))

        # New files to fetch
        for f in new_files:
            channel, filename = f['path'].split('/', 1)

            download = Download (self.config, self.download_dir, self.keys, self.id, channel, filename,
                                 remote_filesize   = f['size'],
                                 callback_step     = self.download_step,
                                 callback_finished = self.download_finished)
            download.execute()
예제 #5
0
def set_md5_attr(fullpath, force=False):
    # Check the file attrs
    attrs = xattr.listxattr(fullpath) or {}

    set_md5 = not 'md5' in attrs or not 'md5_time' in attrs
    set_md5 = set_md5 or force

    # Update
    if not set_md5:
        if 'md5_time' in attrs:
            mtime = os.path.getmtime(fullpath)
            attr_time = xattr.getxattr(fullpath, 'md5_time')

            if mtime > attr_time:
                set_md5 = True
        else:
            set_md5 = True

    if set_md5:
        xattr.setxattr(fullpath, 'md5_time', str(time.time()))
        xattr.setxattr(fullpath, 'md5', md5_file(fullpath))
예제 #6
0
파일: utils.py 프로젝트: pasalo/common
def set_md5_attr (fullpath, force=False):
    # Check the file attrs
    attrs = xattr.listxattr (fullpath) or {}

    set_md5 = not 'md5' in attrs or not 'md5_time' in attrs
    set_md5 = set_md5 or force

    # Update
    if not set_md5:
        if 'md5_time' in attrs:
            mtime     = os.path.getmtime (fullpath)
            attr_time = xattr.getxattr (fullpath, 'md5_time')

            if mtime > attr_time:
                set_md5 = True
        else:
            set_md5 = True

    if set_md5:
        xattr.setxattr (fullpath, 'md5_time', str(time.time()))
        xattr.setxattr (fullpath, 'md5', md5_file(fullpath))
예제 #7
0
    def get_local_files(self, channels):
        downloads_dir = self.config.config['downloads']

        files = []
        for channel in channels:
            if '..' in channel:
                continue

            dir_fp = os.path.join(downloads_dir, channel)

            if not os.path.exists(dir_fp):
                continue
            if not os.path.isdir(dir_fp):
                continue

            for f in os.listdir(dir_fp):
                if f[0] in '.#~':
                    continue

                fname = os.path.join(channel, f)
                fname_fp = os.path.join(dir_fp, f)
                fname_size = os.path.getsize(fname_fp)

                # No dirs within channels
                if not os.path.isfile(fname_fp):
                    continue

                utils.set_md5_attr(fname_fp, force=False)
                fname_md5 = xattr.getxattr(fname_fp, 'md5')

                files.append({
                    'path': fname,
                    'size': fname_size,
                    'md5': fname_md5
                })

        return files