def get_download_url(conf, fileid): """the api to download just redirects to the real url to download :conf: configuration object :fileid: fileid to download :returns: the download url """ LOG.debug( 'trying to dereference download url for file id: %s', str(fileid)) try: conn = httplib.HTTPSConnection('api.put.io') url = API_URL + \ '/files/%s/download?oauth_token=' + conf.get('oauthtoken') url = url % fileid conn.request("GET", url, None, {'User-Agent': USER_AGENT}) response = conn.getresponse() if response.status == 302: return response.getheader('Location') else: LOG.error( 'putio api returned status %d for download: %s', response.status, url) return None except (httplib.HTTPException, IOError, OSError): LOG.error( 'error dereferencing download url for file id: %s', str(fileid), exc_info=True) return None
def exit_invalid_oauth_token(): """exits the process with __INVALID_OAUTH_TOKEN :returns: None """ LOG.error('invalid oauth token') print 'invalid oauth token, please check your configuration.' sys.exit(__INVALID_OAUTH_TOKEN)
def exit_cant_sync_locally(localdir): """exits the process with __ROOT_MIRROR_CANT_BE_SYNCED :returns: None """ LOG.error('local root cant be synced: %s', localdir) print 'cant sync into %s' % localdir sys.exit(__ROOT_MIRROR_CANT_BE_SYNCED)
def exit_interrupted(): """exits the process with __INTERRUPTED :returns: None """ LOG.error('interrupted by user', exc_info=True) print 'interrupted, exiting.' sys.exit(__INTERRUPTED)
def gpg_call_error(message): """exits the process with __GPG_CALL_ERROR :returns: None """ LOG.error('gpg returned with error, perhaps wrong passphrase? %s', message) print message sys.exit(__GPG_CALL_ERROR)
def gpg_not_installed(): """exits the process with __GPG_NOT_INSTALLED :returns: None """ LOG.error('gpg execution failed: probably doesnt exist', exc_info=True) print 'seems like gpg isnt installed' sys.exit(__GPG_NOT_INSTALLED)
def exit_bad_config(): """exits the process with __CODE_BAD_CONFIG :returns: None """ LOG.error('config file is missing essential app info', exc_info=True) print 'config file is missing put.io application info, cant proceed.' sys.exit(__CODE_BAD_CONFIG)
def __create_local_dirs(root, dirtree, files_to_dirs): """creates the local dir tree :conf: configuration object :dirtree: the tree fetched from the putio account :files_to_dirs: a mapping of file data to the dir the file should be downloaded to :returns: None """ todelete = os.listdir(root) for name, remoteitem in dirtree.iteritems(): if name in todelete: todelete.remove(name) if remoteitem is None: print ('Skipping dir %s because no data for it', name) LOG.error('skipping dir %s because no data for it', name) continue target = os.path.join(root, name) if remoteitem.isdir(): LOG.debug('inspecting dir: %s', name) # this is a directory if os.path.exists(target) and not os.path.isdir(target): LOG.warn( "remote dir and local file conflict" + "removing local file: %s", target) os.remove(target) if not os.path.exists(target): LOG.debug('creating dir: %s', target) os.makedirs(target) if remoteitem.dirtree: __create_local_dirs(target, remoteitem.dirtree, files_to_dirs) elif os.path.exists(target): todelete.append(name) else: LOG.debug('inspecting file: %s', name) # this is a normal file exists = os.path.exists(target) if exists and os.path.getsize(target) != remoteitem.size: LOG.warn('file size != from whats on putio: %s', target) todelete.append(name) files_to_dirs[remoteitem] = target elif not exists: LOG.debug( 'file will be downloaded: %s -> %s', remoteitem, target) files_to_dirs[remoteitem] = target sync_utils.delete_files(root, todelete)
def __create_local_dirs(root, dirtree, files_to_dirs): """creates the local dir tree :conf: configuration object :dirtree: the tree fetched from the putio account :files_to_dirs: a mapping of file data to the dir the file should be downloaded to :returns: None """ todelete = os.listdir(root) for name, remoteitem in dirtree.iteritems(): if name in todelete: todelete.remove(name) if remoteitem is None: print('Skipping dir %s because no data for it', name) LOG.error('skipping dir %s because no data for it', name) continue target = os.path.join(root, name) if remoteitem.isdir(): LOG.debug('inspecting dir: %s', name) # this is a directory if os.path.exists(target) and not os.path.isdir(target): LOG.warn( "remote dir and local file conflict" + "removing local file: %s", target) os.remove(target) if not os.path.exists(target): LOG.debug('creating dir: %s', target) os.makedirs(target) if remoteitem.dirtree: __create_local_dirs(target, remoteitem.dirtree, files_to_dirs) elif os.path.exists(target): todelete.append(name) else: LOG.debug('inspecting file: %s', name) # this is a normal file exists = os.path.exists(target) if exists and os.path.getsize(target) != remoteitem.size: LOG.warn('file size != from whats on putio: %s', target) todelete.append(name) files_to_dirs[remoteitem] = target elif not exists: LOG.debug('file will be downloaded: %s -> %s', remoteitem, target) files_to_dirs[remoteitem] = target sync_utils.delete_files(root, todelete)
def make_api_request(conf, resource, params, compress=True): """makes an http call to put.io api :conf: configuration object :resource: the REST resource in the api :params: a dictionary of url parameters key-val :returns: raw response from http response or None if failed """ params['oauth_token'] = conf.get('oauthtoken') url = API_URL + resource + '?' + urlencode(params) LOG.debug('making http request: %s', url) req = urllib2.Request(url) req.add_header('User-Agent', USER_AGENT) req.add_header('Accept', 'application/json') if compress: req.add_header('Accept-Encoding', 'gzip;q=1.0,deflate;q=0.5,*;q=0') try: response = urllib2.urlopen(req) if response.getcode() == 200: content = response.read() try: if compress: try: inflated = gzip.GzipFile( fileobj=StringIO.StringIO(content)) return json.loads(inflated.read()) except IOError: LOG.error( 'request failed due to IO error, content', exc_info=True) return json.loads(content) else: return json.loads(content) except ValueError: LOG.error( 'cant parse api response: %s', content, exc_info=True) return None elif response.getcode() == 302: LOG.debug('got redirect: %s', str(response.info())) return response.info() else: LOG.error('request failed %s status: %s', url, response.getcode()) except urllib2.HTTPError as exc: LOG.error( 'request failed %s status: %s message: %s', url, exc.code, exc.reason) return None
def delete_files(root, files): """deletes files and direcotries that exist locally but not in put.io""" for target in files: abspath = os.path.join(root, target) print '\n [!] Deleting %s since its not in the putio account' % abspath LOG.info('deleting %s since its not in the putio account', abspath) if os.path.exists(abspath): try: if os.path.isdir(abspath): shutil.rmtree(abspath) else: os.remove(abspath) except (OSError, ValueError): print '\n [E] Cant delete dir %s' % abspath LOG.error('cant delete dir %s', abspath, exc_info=True) else: LOG.error('wanted to delete %s but it no longer exists', abspath)
def __sync_account(conf): """perfoms the putio sync action :conf: configuration object :returns: None """ print '\n------------------' print '\n...Sync started...' print '\n------------------' localdir = conf.get('localdir') if os.path.exists(localdir) and not os.path.isdir(localdir): exit_helper.exit_cant_sync_locally(localdir) if not os.path.exists(localdir): os.makedirs(localdir) putio_dirtree = {} while True: try: sync_utils.suspend_until_can_store_all(conf) LOG.info('sync loop iteration started') files_to_dirs = {} putio_dirtree = __get_putio_files(conf, 0, putio_dirtree) __create_local_dirs( conf.get('localdir'), putio_dirtree, files_to_dirs) for remoteitem, targetdir in files_to_dirs.iteritems(): fileid = remoteitem.itemid filesize = remoteitem.size download_url = putio_api.get_download_url(conf, fileid) if download_url: sync_utils.start_download( filesize, download_url, targetdir, conf) except Exception: LOG.error('sync iteration failed', exc_info=True) timenow = __gettimenow() print '\n%s :. waiting...' % timenow sync_utils.suspend_sync()
def delete_files(root, files): """deletes files and direcotries that exist locally but not in put.io""" for target in files: abspath = os.path.join(root, target) print '\n [!] Deleting %s since its not in the putio account' % abspath LOG.info('deleting %s since its not in the putio account', abspath) if os.path.exists(abspath): try: if os.path.isdir(abspath): shutil.rmtree(abspath) else: os.remove(abspath) except (OSError, ValueError): print '\n [E] Cant delete dir %s' % abspath LOG.error('cant delete dir %s', abspath, exc_info=True) else: LOG.error( 'wanted to delete %s but it no longer exists', abspath)
def __sync_account(conf): """perfoms the putio sync action :conf: configuration object :returns: None """ print '\n------------------' print '\n...Sync started...' print '\n------------------' localdir = conf.get('localdir') if os.path.exists(localdir) and not os.path.isdir(localdir): exit_helper.exit_cant_sync_locally(localdir) if not os.path.exists(localdir): os.makedirs(localdir) putio_dirtree = {} while True: try: sync_utils.suspend_until_can_store_all(conf) LOG.info('sync loop iteration started') files_to_dirs = {} putio_dirtree = __get_putio_files(conf, 0, putio_dirtree) __create_local_dirs(conf.get('localdir'), putio_dirtree, files_to_dirs) for remoteitem, targetdir in files_to_dirs.iteritems(): fileid = remoteitem.itemid filesize = remoteitem.size download_url = putio_api.get_download_url(conf, fileid) if download_url: sync_utils.start_download(filesize, download_url, targetdir, conf) except Exception: LOG.error('sync iteration failed', exc_info=True) timenow = __gettimenow() print '\n%s :. waiting...' % timenow sync_utils.suspend_sync()
def start_download(filesize, download_url, targetfile, conf): """downloads the file""" suspend_until_can_store_file(filesize, targetfile) bps = conf.get('bytes_per_second') connections = conf.get('conn_per_downloads') print '\nStarting download :%s' % download_url LOG.info('starting download :%s into %s', download_url, targetfile) cmd = 'axel -o %s -n %d -a -s %d %s' % (targetfile, connections, bps, download_url) LOG.debug('running axel: %s', cmd) axel = subprocess.Popen([ 'axel', '-o', targetfile, '-a', '-s', str(bps), '-n', str(connections), download_url ]) currsize = os.path.getsize(targetfile) if os.path.exists(targetfile) else 0 pollinterval = 5 time.sleep(pollinterval) remaining_attempts = 3 while axel.poll() is None: time.sleep(pollinterval) progress = os.path.getsize(targetfile) - currsize currsize = currsize + progress if progress == 0: LOG.warn('seems like axel isnt effective in the last %d seconds', pollinterval) if remaining_attempts == 0: LOG.error('axel seems totally stuck, aborting') axel.kill() return remaining_attempts = remaining_attempts - 1 pollinterval = pollinterval * 2 returncode = axel.poll() if returncode != 0: print '\n[E] Download %s failed!' % download_url LOG.error('download %s failed with code: %d', download_url, returncode) return if os.path.exists(targetfile): if os.path.getsize(targetfile) != filesize: LOG.info('detected partial download %s due to file size', targetfile) try: os.remove(targetfile) except (OSError, IOError): print '\n[E] Cant remove bad download %s' % targetfile LOG.error('cant remove bad download %s', targetfile, exc_info=True)
def start_download(filesize, download_url, targetfile, conf): """downloads the file""" suspend_until_can_store_file(filesize, targetfile) bps = conf.get('bytes_per_second') connections = conf.get('conn_per_downloads') print '\nStarting download :%s' % download_url LOG.info('starting download :%s into %s', download_url, targetfile) cmd = 'axel -o %s -n %d -a -s %d %s' % (targetfile, connections, bps, download_url) LOG.debug('running axel: %s', cmd) axel = subprocess.Popen( ['axel', '-o', targetfile, '-a', '-s', str(bps), '-n', str(connections), download_url]) currsize = os.path.getsize(targetfile) if os.path.exists(targetfile) else 0 pollinterval = 5 time.sleep(pollinterval) remaining_attempts = 3 while axel.poll() is None: time.sleep(pollinterval) progress = os.path.getsize(targetfile) - currsize currsize = currsize + progress if progress == 0: LOG.warn('seems like axel isnt effective in the last %d seconds', pollinterval) if remaining_attempts == 0: LOG.error('axel seems totally stuck, aborting') axel.kill() return remaining_attempts = remaining_attempts - 1 pollinterval = pollinterval * 2 returncode = axel.poll() if returncode != 0: print '\n[E] Download %s failed!' % download_url LOG.error( 'download %s failed with code: %d', download_url, returncode) return if os.path.exists(targetfile): if os.path.getsize(targetfile) != filesize: LOG.info( 'detected partial download %s due to file size', targetfile) try: os.remove(targetfile) except (OSError, IOError): print '\n[E] Cant remove bad download %s' % targetfile LOG.error( 'cant remove bad download %s', targetfile, exc_info=True)