def __create_local_dirs(root, dirtree, files_to_dirs):
"""creates the local dir tree
:conf: configuration object
:dirtree: the tree fetched from the putio account
:files_to_dirs: a mapping of file data to the dir the file should
be downloaded to
:returns: None
"""
todelete = os.listdir(root)
for name, remoteitem in dirtree.iteritems():
if name in todelete:
todelete.remove(name)
if remoteitem is None:
print ('Skipping dir %s because no data for it', name)
LOG.error('skipping dir %s because no data for it', name)
continue
target = os.path.join(root, name)
if remoteitem.isdir():
LOG.debug('inspecting dir: %s', name)
# this is a directory
if os.path.exists(target) and not os.path.isdir(target):
LOG.warn(
"remote dir and local file conflict" +
"removing local file: %s",
target)
os.remove(target)
if not os.path.exists(target):
LOG.debug('creating dir: %s', target)
os.makedirs(target)
if remoteitem.dirtree:
__create_local_dirs(target, remoteitem.dirtree, files_to_dirs)
elif os.path.exists(target):
todelete.append(name)
else:
LOG.debug('inspecting file: %s', name)
# this is a normal file
exists = os.path.exists(target)
if exists and os.path.getsize(target) != remoteitem.size:
LOG.warn('file size != from whats on putio: %s', target)
todelete.append(name)
files_to_dirs[remoteitem] = target
elif not exists:
LOG.debug(
'file will be downloaded: %s -> %s',
remoteitem,
target)
files_to_dirs[remoteitem] = target
sync_utils.delete_files(root, todelete)
def suspend_until_can_store_file(filesize, targetfile):
"""suspends execution until filesystem can store targetfile"""
while True:
if total_free_space() - min_space_to_reserve() < filesize:
print '\n[!] Suspending Sync: not enough free space to download:\n %s' % targetfile
LOG.warn('not enough free space to download: %s', targetfile)
suspend_sync()
else:
break
def suspend_until_can_store_file(filesize, targetfile):
"""suspends execution until filesystem can store targetfile"""
while True:
if total_free_space() - min_space_to_reserve() < filesize:
print '\n[!] Suspending Sync: not enough free space to download:\n %s' % targetfile
LOG.warn('not enough free space to download: %s',
targetfile)
suspend_sync()
else:
break
def __create_local_dirs(root, dirtree, files_to_dirs):
"""creates the local dir tree
:conf: configuration object
:dirtree: the tree fetched from the putio account
:files_to_dirs: a mapping of file data to the dir the file should
be downloaded to
:returns: None
"""
todelete = os.listdir(root)
for name, remoteitem in dirtree.iteritems():
if name in todelete:
todelete.remove(name)
if remoteitem is None:
print('Skipping dir %s because no data for it', name)
LOG.error('skipping dir %s because no data for it', name)
continue
target = os.path.join(root, name)
if remoteitem.isdir():
LOG.debug('inspecting dir: %s', name)
# this is a directory
if os.path.exists(target) and not os.path.isdir(target):
LOG.warn(
"remote dir and local file conflict" +
"removing local file: %s", target)
os.remove(target)
if not os.path.exists(target):
LOG.debug('creating dir: %s', target)
os.makedirs(target)
if remoteitem.dirtree:
__create_local_dirs(target, remoteitem.dirtree, files_to_dirs)
elif os.path.exists(target):
todelete.append(name)
else:
LOG.debug('inspecting file: %s', name)
# this is a normal file
exists = os.path.exists(target)
if exists and os.path.getsize(target) != remoteitem.size:
LOG.warn('file size != from whats on putio: %s', target)
todelete.append(name)
files_to_dirs[remoteitem] = target
elif not exists:
LOG.debug('file will be downloaded: %s -> %s', remoteitem,
target)
files_to_dirs[remoteitem] = target
sync_utils.delete_files(root, todelete)
def start_download(filesize, download_url, targetfile, conf):
"""downloads the file"""
suspend_until_can_store_file(filesize, targetfile)
bps = conf.get('bytes_per_second')
connections = conf.get('conn_per_downloads')
print '\nStarting download :%s' % download_url
LOG.info('starting download :%s into %s', download_url, targetfile)
cmd = 'axel -o %s -n %d -a -s %d %s' % (targetfile, connections, bps,
download_url)
LOG.debug('running axel: %s', cmd)
axel = subprocess.Popen([
'axel', '-o', targetfile, '-a', '-s',
str(bps), '-n',
str(connections), download_url
])
currsize = os.path.getsize(targetfile) if os.path.exists(targetfile) else 0
pollinterval = 5
time.sleep(pollinterval)
remaining_attempts = 3
while axel.poll() is None:
time.sleep(pollinterval)
progress = os.path.getsize(targetfile) - currsize
currsize = currsize + progress
if progress == 0:
LOG.warn('seems like axel isnt effective in the last %d seconds',
pollinterval)
if remaining_attempts == 0:
LOG.error('axel seems totally stuck, aborting')
axel.kill()
return
remaining_attempts = remaining_attempts - 1
pollinterval = pollinterval * 2
returncode = axel.poll()
if returncode != 0:
print '\n[E] Download %s failed!' % download_url
LOG.error('download %s failed with code: %d', download_url, returncode)
return
if os.path.exists(targetfile):
if os.path.getsize(targetfile) != filesize:
LOG.info('detected partial download %s due to file size',
targetfile)
try:
os.remove(targetfile)
except (OSError, IOError):
print '\n[E] Cant remove bad download %s' % targetfile
LOG.error('cant remove bad download %s',
targetfile,
exc_info=True)
def suspend_until_can_store_all(conf):
"""ensures local filesystem have enough space on disk to sync
:conf: configuration object
:returns:
"""
LOG.info('ensuring enough space in filesystem')
while True:
localsize = os.path.getsize(conf.get('localdir'))
# we don't account for local size because it's replaced if necessary
free_space = total_free_space() + localsize
putio_size = putio_root_size(conf)
if free_space - min_space_to_reserve() < putio_size:
print '\n[!] Suspending Sync: not enough space to sync local: %d remote: %d ' % free_space, putio_size
LOG.warn('not enough space to sync local: %d remote: %d',
free_space, putio_size)
suspend_sync()
else:
break
def suspend_until_can_store_all(conf):
"""ensures local filesystem have enough space on disk to sync
:conf: configuration object
:returns:
"""
LOG.info('ensuring enough space in filesystem')
while True:
localsize = os.path.getsize(conf.get('localdir'))
# we don't account for local size because it's replaced if necessary
free_space = total_free_space() + localsize
putio_size = putio_root_size(conf)
if free_space - min_space_to_reserve() < putio_size:
print '\n[!] Suspending Sync: not enough space to sync local: %d remote: %d ' % free_space,putio_size
LOG.warn('not enough space to sync local: %d remote: %d',
free_space,
putio_size)
suspend_sync()
else:
break
def start_download(filesize, download_url, targetfile, conf):
"""downloads the file"""
suspend_until_can_store_file(filesize, targetfile)
bps = conf.get('bytes_per_second')
connections = conf.get('conn_per_downloads')
print '\nStarting download :%s' % download_url
LOG.info('starting download :%s into %s', download_url, targetfile)
cmd = 'axel -o %s -n %d -a -s %d %s' % (targetfile,
connections,
bps,
download_url)
LOG.debug('running axel: %s', cmd)
axel = subprocess.Popen(
['axel',
'-o',
targetfile,
'-a',
'-s',
str(bps),
'-n',
str(connections),
download_url])
currsize = os.path.getsize(targetfile) if os.path.exists(targetfile) else 0
pollinterval = 5
time.sleep(pollinterval)
remaining_attempts = 3
while axel.poll() is None:
time.sleep(pollinterval)
progress = os.path.getsize(targetfile) - currsize
currsize = currsize + progress
if progress == 0:
LOG.warn('seems like axel isnt effective in the last %d seconds',
pollinterval)
if remaining_attempts == 0:
LOG.error('axel seems totally stuck, aborting')
axel.kill()
return
remaining_attempts = remaining_attempts - 1
pollinterval = pollinterval * 2
returncode = axel.poll()
if returncode != 0:
print '\n[E] Download %s failed!' % download_url
LOG.error(
'download %s failed with code: %d',
download_url,
returncode)
return
if os.path.exists(targetfile):
if os.path.getsize(targetfile) != filesize:
LOG.info(
'detected partial download %s due to file size',
targetfile)
try:
os.remove(targetfile)
except (OSError, IOError):
print '\n[E] Cant remove bad download %s' % targetfile
LOG.error(
'cant remove bad download %s',
targetfile,
exc_info=True)