def get_local_files (self, channels):
downloads_dir = self.config.config['downloads']
files = []
for channel in channels:
if '..' in channel:
continue
dir_fp = os.path.join (downloads_dir, channel)
if not os.path.exists (dir_fp):
continue
if not os.path.isdir (dir_fp):
continue
for f in os.listdir (dir_fp):
if f[0] in '.#~':
continue
fname = os.path.join (channel, f)
fname_fp = os.path.join (dir_fp, f)
fname_size = os.path.getsize(fname_fp)
# No dirs within channels
if not os.path.isfile (fname_fp):
continue
utils.set_md5_attr (fname_fp, force=False)
fname_md5 = xattr.getxattr (fname_fp, 'md5')
files.append ({'path': fname, 'size': fname_size, 'md5': fname_md5})
return files
def getxattr(fullpath, attr, default=None):
try:
return xattr.getxattr(fullpath, attr)
except IOError, e:
# Attribute not found
if e.errno == 93:
return default
raise
def getxattr (fullpath, attr, default=None):
try:
return xattr.getxattr (fullpath, attr)
except IOError, e:
# Attribute not found
if e.errno == 93:
return default
raise
def execute (self):
# Channels
lst = ChannelList (self.config, self.keys, self.id)
channels = lst.execute()
# Files
lst = FileList (self.config, self.keys, self.id, ','.join(channels))
remote_files = lst.execute()
# Check local files
new_files = []
for remote_file in remote_files:
fp = os.path.join (self.download_dir, remote_file['path'])
# No local version
if not os.path.exists (fp):
new_files.append (remote_file)
continue
# Outdated local version
size = os.path.getsize(fp)
attr_md5 = xattr.getxattr (fp, 'md5')
attr_time = utils.getxattr (fp, 'md5_time', 0)
if remote_file['size'] != size:
new_files.append (remote_file)
continue
if remote_file['md5'] != attr_md5:
new_files.append (remote_file)
continue
logging.info ("%s is up to date" %(remote_file['path']))
if not new_files:
return
# Report
for f in new_files:
channel, filename = f['path'].split('/', 1)
print (' #%s - %s (%s)' %(channel, filename, utils.format_size(f['size'])))
total_size = reduce (lambda x,y: x+y, [f['size'] for f in new_files])
print ("%d files: %s"%(len(new_files), utils.format_size(total_size)))
# New files to fetch
for f in new_files:
channel, filename = f['path'].split('/', 1)
download = Download (self.config, self.download_dir, self.keys, self.id, channel, filename,
remote_filesize = f['size'],
callback_step = self.download_step,
callback_finished = self.download_finished)
download.execute()
def set_md5_attr(fullpath, force=False):
# Check the file attrs
attrs = xattr.listxattr(fullpath) or {}
set_md5 = not 'md5' in attrs or not 'md5_time' in attrs
set_md5 = set_md5 or force
# Update
if not set_md5:
if 'md5_time' in attrs:
mtime = os.path.getmtime(fullpath)
attr_time = xattr.getxattr(fullpath, 'md5_time')
if mtime > attr_time:
set_md5 = True
else:
set_md5 = True
if set_md5:
xattr.setxattr(fullpath, 'md5_time', str(time.time()))
xattr.setxattr(fullpath, 'md5', md5_file(fullpath))
def set_md5_attr (fullpath, force=False):
# Check the file attrs
attrs = xattr.listxattr (fullpath) or {}
set_md5 = not 'md5' in attrs or not 'md5_time' in attrs
set_md5 = set_md5 or force
# Update
if not set_md5:
if 'md5_time' in attrs:
mtime = os.path.getmtime (fullpath)
attr_time = xattr.getxattr (fullpath, 'md5_time')
if mtime > attr_time:
set_md5 = True
else:
set_md5 = True
if set_md5:
xattr.setxattr (fullpath, 'md5_time', str(time.time()))
xattr.setxattr (fullpath, 'md5', md5_file(fullpath))
def get_local_files(self, channels):
downloads_dir = self.config.config['downloads']
files = []
for channel in channels:
if '..' in channel:
continue
dir_fp = os.path.join(downloads_dir, channel)
if not os.path.exists(dir_fp):
continue
if not os.path.isdir(dir_fp):
continue
for f in os.listdir(dir_fp):
if f[0] in '.#~':
continue
fname = os.path.join(channel, f)
fname_fp = os.path.join(dir_fp, f)
fname_size = os.path.getsize(fname_fp)
# No dirs within channels
if not os.path.isfile(fname_fp):
continue
utils.set_md5_attr(fname_fp, force=False)
fname_md5 = xattr.getxattr(fname_fp, 'md5')
files.append({
'path': fname,
'size': fname_size,
'md5': fname_md5
})
return files