def exit_interrupted():
"""exits the process with __INTERRUPTED
:returns: None
"""
LOG.error('interrupted by user', exc_info=True)
print 'interrupted, exiting.'
sys.exit(__INTERRUPTED)
def gpg_not_installed():
"""exits the process with __GPG_NOT_INSTALLED
:returns: None
"""
LOG.error('gpg execution failed: probably doesnt exist', exc_info=True)
print 'seems like gpg isnt installed'
sys.exit(__GPG_NOT_INSTALLED)
def gpg_call_error(message):
"""exits the process with __GPG_CALL_ERROR
:returns: None
"""
LOG.error('gpg returned with error, perhaps wrong passphrase? %s', message)
print message
sys.exit(__GPG_CALL_ERROR)
def exit_invalid_oauth_token():
"""exits the process with __INVALID_OAUTH_TOKEN
:returns: None
"""
LOG.error('invalid oauth token')
print 'invalid oauth token, please check your configuration.'
sys.exit(__INVALID_OAUTH_TOKEN)
def exit_cant_sync_locally(localdir):
"""exits the process with __ROOT_MIRROR_CANT_BE_SYNCED
:returns: None
"""
LOG.error('local root cant be synced: %s', localdir)
print 'cant sync into %s' % localdir
sys.exit(__ROOT_MIRROR_CANT_BE_SYNCED)
def exit_bad_config():
"""exits the process with __CODE_BAD_CONFIG
:returns: None
"""
LOG.error('config file is missing essential app info', exc_info=True)
print 'config file is missing put.io application info, cant proceed.'
sys.exit(__CODE_BAD_CONFIG)
def suspend_until_can_store_file(filesize, targetfile):
"""suspends execution until filesystem can store targetfile"""
while True:
if total_free_space() - min_space_to_reserve() < filesize:
print '\n[!] Suspending Sync: not enough free space to download:\n %s' % targetfile
LOG.warn('not enough free space to download: %s', targetfile)
suspend_sync()
else:
break
def suspend_until_can_store_file(filesize, targetfile):
"""suspends execution until filesystem can store targetfile"""
while True:
if total_free_space() - min_space_to_reserve() < filesize:
print '\n[!] Suspending Sync: not enough free space to download:\n %s' % targetfile
LOG.warn('not enough free space to download: %s',
targetfile)
suspend_sync()
else:
break
def __readargs():
try:
conf = __getconfig()
LOG.info('starting sync in home dir: %s', conf.get('localdir'))
print '\n------------------'
print "... PutIO/Sync ..."
print '\n------------------'
print "Minimum Reserved: \t%i" % sync_utils.min_space_to_reserve()
print "Available Disk Space: \t%i" % sync_utils.total_free_space()
print "Local Sync Dir: \t" + conf.get('localdir')
__sync_account(conf)
except KeyboardInterrupt:
exit_helper.exit_interrupted()
def __readargs():
try:
conf = __getconfig()
LOG.info('starting sync in home dir: %s', conf.get('localdir'))
print '\n------------------'
print "... PutIO/Sync ..."
print '\n------------------'
print "Minimum Reserved: \t%i" % sync_utils.min_space_to_reserve()
print "Available Disk Space: \t%i" % sync_utils.total_free_space()
print "Local Sync Dir: \t"+conf.get('localdir')
__sync_account(conf)
except KeyboardInterrupt:
exit_helper.exit_interrupted()
def __sync_account(conf):
"""perfoms the putio sync action
:conf: configuration object
:returns: None
"""
print '\n------------------'
print '\n...Sync started...'
print '\n------------------'
localdir = conf.get('localdir')
if os.path.exists(localdir) and not os.path.isdir(localdir):
exit_helper.exit_cant_sync_locally(localdir)
if not os.path.exists(localdir):
os.makedirs(localdir)
putio_dirtree = {}
while True:
try:
sync_utils.suspend_until_can_store_all(conf)
LOG.info('sync loop iteration started')
files_to_dirs = {}
putio_dirtree = __get_putio_files(conf, 0, putio_dirtree)
__create_local_dirs(
conf.get('localdir'),
putio_dirtree,
files_to_dirs)
for remoteitem, targetdir in files_to_dirs.iteritems():
fileid = remoteitem.itemid
filesize = remoteitem.size
download_url = putio_api.get_download_url(conf, fileid)
if download_url:
sync_utils.start_download(
filesize,
download_url,
targetdir,
conf)
except Exception:
LOG.error('sync iteration failed', exc_info=True)
timenow = __gettimenow()
print '\n%s :. waiting...' % timenow
sync_utils.suspend_sync()
def __get_putio_files(conf, parent_id=0, tree=None, root=None):
"""fetches the file list from put.io account
:conf: configuration object
:parent_id: the from which to start on the account
:returns: a dict of dicts representing the account directory tree
"""
if not tree:
tree = {}
if not root:
root = '/'
data = putio_api.getfiles(conf, parent_id)
putio_api.ensure_valid_oauth_token(data)
freshtree = {}
if data:
LOG.debug('got data for file id: %d', parent_id)
for remotefile in data.get('files'):
filename = remotefile.get('name')
filetype = remotefile.get('file_type')
fileid = remotefile.get('id')
filesize = remotefile.get('size')
abspath = root + '/' + filename
skip = False
for exclude in EXCLUDE_LIST:
skip = abspath == exclude or re.search(exclude, abspath)
if skip:
LOG.info('skipping because exclude rule match (%s ~ %s)',
exclude, abspath)
break
if skip:
continue
if filetype == PUTIO_DIR_FTP:
cached = tree.get(filename, None)
cached_filesize = cached.size if cached else 0
if cached:
freshtree[filename] = cached
if filesize != cached_filesize:
subtree = cached.dirtree if cached else {}
subtree = __get_putio_files(conf, fileid, subtree, abspath)
freshtree[filename] = RemoteItem(
filename, filesize, fileid, subtree)
LOG.debug('mapped directory: %s', freshtree[filename])
else:
filedata = RemoteItem(filename, filesize, fileid, None)
LOG.debug('mapped file: %s', filedata)
freshtree[filename] = filedata
tree = freshtree
return tree
def __get_putio_files(conf, parent_id=0, tree=None, root=None):
"""fetches the file list from put.io account
:conf: configuration object
:parent_id: the from which to start on the account
:returns: a dict of dicts representing the account directory tree
"""
if not tree:
tree = {}
if not root:
root = '/'
data = putio_api.getfiles(conf, parent_id)
putio_api.ensure_valid_oauth_token(data)
freshtree = {}
if data:
LOG.debug('got data for file id: %d', parent_id)
for remotefile in data.get('files'):
filename = remotefile.get('name')
filetype = remotefile.get('file_type')
fileid = remotefile.get('id')
filesize = remotefile.get('size')
abspath = root + '/' + filename
skip = False
for exclude in EXCLUDE_LIST:
skip = abspath == exclude or re.search(exclude, abspath)
if skip:
LOG.info('skipping because exclude rule match (%s ~ %s)',
exclude, abspath)
break
if skip:
continue
if filetype == PUTIO_DIR_FTP:
cached = tree.get(filename, None)
cached_filesize = cached.size if cached else 0
if cached:
freshtree[filename] = cached
if filesize != cached_filesize:
subtree = cached.dirtree if cached else {}
subtree = __get_putio_files(conf, fileid, subtree, abspath)
freshtree[filename] = RemoteItem(filename, filesize,
fileid, subtree)
LOG.debug('mapped directory: %s', freshtree[filename])
else:
filedata = RemoteItem(filename, filesize, fileid, None)
LOG.debug('mapped file: %s', filedata)
freshtree[filename] = filedata
tree = freshtree
return tree
def suspend_until_can_store_all(conf):
"""ensures local filesystem have enough space on disk to sync
:conf: configuration object
:returns:
"""
LOG.info('ensuring enough space in filesystem')
while True:
localsize = os.path.getsize(conf.get('localdir'))
# we don't account for local size because it's replaced if necessary
free_space = total_free_space() + localsize
putio_size = putio_root_size(conf)
if free_space - min_space_to_reserve() < putio_size:
print '\n[!] Suspending Sync: not enough space to sync local: %d remote: %d ' % free_space, putio_size
LOG.warn('not enough space to sync local: %d remote: %d',
free_space, putio_size)
suspend_sync()
else:
break
def suspend_until_can_store_all(conf):
"""ensures local filesystem have enough space on disk to sync
:conf: configuration object
:returns:
"""
LOG.info('ensuring enough space in filesystem')
while True:
localsize = os.path.getsize(conf.get('localdir'))
# we don't account for local size because it's replaced if necessary
free_space = total_free_space() + localsize
putio_size = putio_root_size(conf)
if free_space - min_space_to_reserve() < putio_size:
print '\n[!] Suspending Sync: not enough space to sync local: %d remote: %d ' % free_space,putio_size
LOG.warn('not enough space to sync local: %d remote: %d',
free_space,
putio_size)
suspend_sync()
else:
break
def __sync_account(conf):
"""perfoms the putio sync action
:conf: configuration object
:returns: None
"""
print '\n------------------'
print '\n...Sync started...'
print '\n------------------'
localdir = conf.get('localdir')
if os.path.exists(localdir) and not os.path.isdir(localdir):
exit_helper.exit_cant_sync_locally(localdir)
if not os.path.exists(localdir):
os.makedirs(localdir)
putio_dirtree = {}
while True:
try:
sync_utils.suspend_until_can_store_all(conf)
LOG.info('sync loop iteration started')
files_to_dirs = {}
putio_dirtree = __get_putio_files(conf, 0, putio_dirtree)
__create_local_dirs(conf.get('localdir'), putio_dirtree,
files_to_dirs)
for remoteitem, targetdir in files_to_dirs.iteritems():
fileid = remoteitem.itemid
filesize = remoteitem.size
download_url = putio_api.get_download_url(conf, fileid)
if download_url:
sync_utils.start_download(filesize, download_url,
targetdir, conf)
except Exception:
LOG.error('sync iteration failed', exc_info=True)
timenow = __gettimenow()
print '\n%s :. waiting...' % timenow
sync_utils.suspend_sync()
def __getconfig():
"""creates a config object used later in the script
:returns: dictionary with the config
"""
if not OAUTH_TOKEN:
exit_helper.exit_bad_config()
oauthtoken = OAUTH_TOKEN
if OAUTH_TOKEN_SYMMETRIC_ARMOR_BASE64:
LOG.info('app info is encrypted, prompting gpg passphrase')
print 'app info is encrypted, running gpg to decrypt'
oauthtoken = sync_utils.gpgdecode(oauthtoken)
oauthtoken = oauthtoken.strip('\n ')
return dict(oauthtoken=oauthtoken,
parallel_downloads=PARALLEL_DOWNLOADS,
conn_per_downloads=CONNECTIONS_PER_DOWNLOAD,
localdir=LOCAL_MIRROR_ROOT,
bytes_per_second=MAX_DOWNLOAD_SPEED_BYTES_PER_SECOND)
def __getconfig():
"""creates a config object used later in the script
:returns: dictionary with the config
"""
if not OAUTH_TOKEN:
exit_helper.exit_bad_config()
oauthtoken = OAUTH_TOKEN
if OAUTH_TOKEN_SYMMETRIC_ARMOR_BASE64:
LOG.info('app info is encrypted, prompting gpg passphrase')
print 'app info is encrypted, running gpg to decrypt'
oauthtoken = sync_utils.gpgdecode(oauthtoken)
oauthtoken = oauthtoken.strip('\n ')
return dict(oauthtoken=oauthtoken,
parallel_downloads=PARALLEL_DOWNLOADS,
conn_per_downloads=CONNECTIONS_PER_DOWNLOAD,
localdir=LOCAL_MIRROR_ROOT,
bytes_per_second=MAX_DOWNLOAD_SPEED_BYTES_PER_SECOND)
def get_download_url(conf, fileid):
"""the api to download just redirects to the real url to download
:conf: configuration object
:fileid: fileid to download
:returns: the download url
"""
LOG.debug(
'trying to dereference download url for file id: %s',
str(fileid))
try:
conn = httplib.HTTPSConnection('api.put.io')
url = API_URL + \
'/files/%s/download?oauth_token=' + conf.get('oauthtoken')
url = url % fileid
conn.request("GET", url, None, {'User-Agent': USER_AGENT})
response = conn.getresponse()
if response.status == 302:
return response.getheader('Location')
else:
LOG.error(
'putio api returned status %d for download: %s',
response.status, url)
return None
except (httplib.HTTPException, IOError, OSError):
LOG.error(
'error dereferencing download url for file id: %s',
str(fileid), exc_info=True)
return None
def __check_filesize_and_crc(targetfile, expected_size, expected_crc32):
"""check a file for expected size and crc32
:targetfile: file to check
:size: expected size
:crc: crc32 checksum
:returns: True if check is ok
"""
LOG.info('doing byte count and crc32 check to file %s', targetfile)
if os.path.getsize(targetfile) != expected_size:
LOG.info('detected partial download %s due to filesize', targetfile)
return False
else:
with open(targetfile, 'r') as binfile:
crc32 = binascii.crc32(binfile.read()) & 0xFFFFFFFF
crchex = "%08X" % crc32
crchex = crchex.lower()
if crchex.encode('utf-8') != expected_crc32.encode('utf-8'):
LOG.info(
'detected partial download due to crc32 got: ' +
'%s expected: %s file: %s', crchex, expected_crc32,
targetfile)
return False
return True
def __check_filesize_and_crc(targetfile, expected_size, expected_crc32):
"""check a file for expected size and crc32
:targetfile: file to check
:size: expected size
:crc: crc32 checksum
:returns: True if check is ok
"""
LOG.info('doing byte count and crc32 check to file %s', targetfile)
if os.path.getsize(targetfile) != expected_size:
LOG.info(
'detected partial download %s due to filesize', targetfile)
return False
else:
with open(targetfile, 'r') as binfile:
crc32 = binascii.crc32(binfile.read()) & 0xFFFFFFFF
crchex = "%08X" % crc32
crchex = crchex.lower()
if crchex.encode('utf-8') != expected_crc32.encode('utf-8'):
LOG.info('detected partial download due to crc32 got: ' +
'%s expected: %s file: %s',
crchex, expected_crc32, targetfile)
return False
return True
def delete_files(root, files):
"""deletes files and direcotries that exist locally but not in put.io"""
for target in files:
abspath = os.path.join(root, target)
print '\n [!] Deleting %s since its not in the putio account' % abspath
LOG.info('deleting %s since its not in the putio account', abspath)
if os.path.exists(abspath):
try:
if os.path.isdir(abspath):
shutil.rmtree(abspath)
else:
os.remove(abspath)
except (OSError, ValueError):
print '\n [E] Cant delete dir %s' % abspath
LOG.error('cant delete dir %s', abspath, exc_info=True)
else:
LOG.error('wanted to delete %s but it no longer exists', abspath)
def delete_files(root, files):
"""deletes files and direcotries that exist locally but not in put.io"""
for target in files:
abspath = os.path.join(root, target)
print '\n [!] Deleting %s since its not in the putio account' % abspath
LOG.info('deleting %s since its not in the putio account',
abspath)
if os.path.exists(abspath):
try:
if os.path.isdir(abspath):
shutil.rmtree(abspath)
else:
os.remove(abspath)
except (OSError, ValueError):
print '\n [E] Cant delete dir %s' % abspath
LOG.error('cant delete dir %s', abspath, exc_info=True)
else:
LOG.error(
'wanted to delete %s but it no longer exists',
abspath)
def __create_local_dirs(root, dirtree, files_to_dirs):
"""creates the local dir tree
:conf: configuration object
:dirtree: the tree fetched from the putio account
:files_to_dirs: a mapping of file data to the dir the file should
be downloaded to
:returns: None
"""
todelete = os.listdir(root)
for name, remoteitem in dirtree.iteritems():
if name in todelete:
todelete.remove(name)
if remoteitem is None:
print('Skipping dir %s because no data for it', name)
LOG.error('skipping dir %s because no data for it', name)
continue
target = os.path.join(root, name)
if remoteitem.isdir():
LOG.debug('inspecting dir: %s', name)
# this is a directory
if os.path.exists(target) and not os.path.isdir(target):
LOG.warn(
"remote dir and local file conflict" +
"removing local file: %s", target)
os.remove(target)
if not os.path.exists(target):
LOG.debug('creating dir: %s', target)
os.makedirs(target)
if remoteitem.dirtree:
__create_local_dirs(target, remoteitem.dirtree, files_to_dirs)
elif os.path.exists(target):
todelete.append(name)
else:
LOG.debug('inspecting file: %s', name)
# this is a normal file
exists = os.path.exists(target)
if exists and os.path.getsize(target) != remoteitem.size:
LOG.warn('file size != from whats on putio: %s', target)
todelete.append(name)
files_to_dirs[remoteitem] = target
elif not exists:
LOG.debug('file will be downloaded: %s -> %s', remoteitem,
target)
files_to_dirs[remoteitem] = target
sync_utils.delete_files(root, todelete)
def __create_local_dirs(root, dirtree, files_to_dirs):
"""creates the local dir tree
:conf: configuration object
:dirtree: the tree fetched from the putio account
:files_to_dirs: a mapping of file data to the dir the file should
be downloaded to
:returns: None
"""
todelete = os.listdir(root)
for name, remoteitem in dirtree.iteritems():
if name in todelete:
todelete.remove(name)
if remoteitem is None:
print ('Skipping dir %s because no data for it', name)
LOG.error('skipping dir %s because no data for it', name)
continue
target = os.path.join(root, name)
if remoteitem.isdir():
LOG.debug('inspecting dir: %s', name)
# this is a directory
if os.path.exists(target) and not os.path.isdir(target):
LOG.warn(
"remote dir and local file conflict" +
"removing local file: %s",
target)
os.remove(target)
if not os.path.exists(target):
LOG.debug('creating dir: %s', target)
os.makedirs(target)
if remoteitem.dirtree:
__create_local_dirs(target, remoteitem.dirtree, files_to_dirs)
elif os.path.exists(target):
todelete.append(name)
else:
LOG.debug('inspecting file: %s', name)
# this is a normal file
exists = os.path.exists(target)
if exists and os.path.getsize(target) != remoteitem.size:
LOG.warn('file size != from whats on putio: %s', target)
todelete.append(name)
files_to_dirs[remoteitem] = target
elif not exists:
LOG.debug(
'file will be downloaded: %s -> %s',
remoteitem,
target)
files_to_dirs[remoteitem] = target
sync_utils.delete_files(root, todelete)
def make_api_request(conf, resource, params, compress=True):
"""makes an http call to put.io api
:conf: configuration object
:resource: the REST resource in the api
:params: a dictionary of url parameters key-val
:returns: raw response from http response or None if failed
"""
params['oauth_token'] = conf.get('oauthtoken')
url = API_URL + resource + '?' + urlencode(params)
LOG.debug('making http request: %s', url)
req = urllib2.Request(url)
req.add_header('User-Agent', USER_AGENT)
req.add_header('Accept', 'application/json')
if compress:
req.add_header('Accept-Encoding', 'gzip;q=1.0,deflate;q=0.5,*;q=0')
try:
response = urllib2.urlopen(req)
if response.getcode() == 200:
content = response.read()
try:
if compress:
try:
inflated = gzip.GzipFile(
fileobj=StringIO.StringIO(content))
return json.loads(inflated.read())
except IOError:
LOG.error(
'request failed due to IO error, content',
exc_info=True)
return json.loads(content)
else:
return json.loads(content)
except ValueError:
LOG.error(
'cant parse api response: %s',
content,
exc_info=True)
return None
elif response.getcode() == 302:
LOG.debug('got redirect: %s', str(response.info()))
return response.info()
else:
LOG.error('request failed %s status: %s', url, response.getcode())
except urllib2.HTTPError as exc:
LOG.error(
'request failed %s status: %s message: %s',
url,
exc.code,
exc.reason)
return None
def start_download(filesize, download_url, targetfile, conf):
"""downloads the file"""
suspend_until_can_store_file(filesize, targetfile)
bps = conf.get('bytes_per_second')
connections = conf.get('conn_per_downloads')
print '\nStarting download :%s' % download_url
LOG.info('starting download :%s into %s', download_url, targetfile)
cmd = 'axel -o %s -n %d -a -s %d %s' % (targetfile, connections, bps,
download_url)
LOG.debug('running axel: %s', cmd)
axel = subprocess.Popen([
'axel', '-o', targetfile, '-a', '-s',
str(bps), '-n',
str(connections), download_url
])
currsize = os.path.getsize(targetfile) if os.path.exists(targetfile) else 0
pollinterval = 5
time.sleep(pollinterval)
remaining_attempts = 3
while axel.poll() is None:
time.sleep(pollinterval)
progress = os.path.getsize(targetfile) - currsize
currsize = currsize + progress
if progress == 0:
LOG.warn('seems like axel isnt effective in the last %d seconds',
pollinterval)
if remaining_attempts == 0:
LOG.error('axel seems totally stuck, aborting')
axel.kill()
return
remaining_attempts = remaining_attempts - 1
pollinterval = pollinterval * 2
returncode = axel.poll()
if returncode != 0:
print '\n[E] Download %s failed!' % download_url
LOG.error('download %s failed with code: %d', download_url, returncode)
return
if os.path.exists(targetfile):
if os.path.getsize(targetfile) != filesize:
LOG.info('detected partial download %s due to file size',
targetfile)
try:
os.remove(targetfile)
except (OSError, IOError):
print '\n[E] Cant remove bad download %s' % targetfile
LOG.error('cant remove bad download %s',
targetfile,
exc_info=True)
def start_download(filesize, download_url, targetfile, conf):
"""downloads the file"""
suspend_until_can_store_file(filesize, targetfile)
bps = conf.get('bytes_per_second')
connections = conf.get('conn_per_downloads')
print '\nStarting download :%s' % download_url
LOG.info('starting download :%s into %s', download_url, targetfile)
cmd = 'axel -o %s -n %d -a -s %d %s' % (targetfile,
connections,
bps,
download_url)
LOG.debug('running axel: %s', cmd)
axel = subprocess.Popen(
['axel',
'-o',
targetfile,
'-a',
'-s',
str(bps),
'-n',
str(connections),
download_url])
currsize = os.path.getsize(targetfile) if os.path.exists(targetfile) else 0
pollinterval = 5
time.sleep(pollinterval)
remaining_attempts = 3
while axel.poll() is None:
time.sleep(pollinterval)
progress = os.path.getsize(targetfile) - currsize
currsize = currsize + progress
if progress == 0:
LOG.warn('seems like axel isnt effective in the last %d seconds',
pollinterval)
if remaining_attempts == 0:
LOG.error('axel seems totally stuck, aborting')
axel.kill()
return
remaining_attempts = remaining_attempts - 1
pollinterval = pollinterval * 2
returncode = axel.poll()
if returncode != 0:
print '\n[E] Download %s failed!' % download_url
LOG.error(
'download %s failed with code: %d',
download_url,
returncode)
return
if os.path.exists(targetfile):
if os.path.getsize(targetfile) != filesize:
LOG.info(
'detected partial download %s due to file size',
targetfile)
try:
os.remove(targetfile)
except (OSError, IOError):
print '\n[E] Cant remove bad download %s' % targetfile
LOG.error(
'cant remove bad download %s',
targetfile,
exc_info=True)
def suspend_sync():
"""simple sleep helper"""
sleep_seconds = 60 * 1
LOG.debug('sleeping for %d seconds', sleep_seconds)
time.sleep(sleep_seconds)
def suspend_sync():
"""simple sleep helper"""
sleep_seconds = 60 * 1
LOG.debug('sleeping for %d seconds', sleep_seconds)
time.sleep(sleep_seconds)