def __get_putio_files(conf, parent_id=0, tree=None, root=None): """fetches the file list from put.io account :conf: configuration object :parent_id: the from which to start on the account :returns: a dict of dicts representing the account directory tree """ if not tree: tree = {} if not root: root = '/' data = putio_api.getfiles(conf, parent_id) putio_api.ensure_valid_oauth_token(data) freshtree = {} if data: LOG.debug('got data for file id: %d', parent_id) for remotefile in data.get('files'): filename = remotefile.get('name') filetype = remotefile.get('file_type') fileid = remotefile.get('id') filesize = remotefile.get('size') abspath = root + '/' + filename skip = False for exclude in EXCLUDE_LIST: skip = abspath == exclude or re.search(exclude, abspath) if skip: LOG.info('skipping because exclude rule match (%s ~ %s)', exclude, abspath) break if skip: continue if filetype == PUTIO_DIR_FTP: cached = tree.get(filename, None) cached_filesize = cached.size if cached else 0 if cached: freshtree[filename] = cached if filesize != cached_filesize: subtree = cached.dirtree if cached else {} subtree = __get_putio_files(conf, fileid, subtree, abspath) freshtree[filename] = RemoteItem( filename, filesize, fileid, subtree) LOG.debug('mapped directory: %s', freshtree[filename]) else: filedata = RemoteItem(filename, filesize, fileid, None) LOG.debug('mapped file: %s', filedata) freshtree[filename] = filedata tree = freshtree return tree
def __get_putio_files(conf, parent_id=0, tree=None, root=None): """fetches the file list from put.io account :conf: configuration object :parent_id: the from which to start on the account :returns: a dict of dicts representing the account directory tree """ if not tree: tree = {} if not root: root = '/' data = putio_api.getfiles(conf, parent_id) putio_api.ensure_valid_oauth_token(data) freshtree = {} if data: LOG.debug('got data for file id: %d', parent_id) for remotefile in data.get('files'): filename = remotefile.get('name') filetype = remotefile.get('file_type') fileid = remotefile.get('id') filesize = remotefile.get('size') abspath = root + '/' + filename skip = False for exclude in EXCLUDE_LIST: skip = abspath == exclude or re.search(exclude, abspath) if skip: LOG.info('skipping because exclude rule match (%s ~ %s)', exclude, abspath) break if skip: continue if filetype == PUTIO_DIR_FTP: cached = tree.get(filename, None) cached_filesize = cached.size if cached else 0 if cached: freshtree[filename] = cached if filesize != cached_filesize: subtree = cached.dirtree if cached else {} subtree = __get_putio_files(conf, fileid, subtree, abspath) freshtree[filename] = RemoteItem(filename, filesize, fileid, subtree) LOG.debug('mapped directory: %s', freshtree[filename]) else: filedata = RemoteItem(filename, filesize, fileid, None) LOG.debug('mapped file: %s', filedata) freshtree[filename] = filedata tree = freshtree return tree
def start_download(filesize, download_url, targetfile, conf): """downloads the file""" suspend_until_can_store_file(filesize, targetfile) bps = conf.get('bytes_per_second') connections = conf.get('conn_per_downloads') print '\nStarting download :%s' % download_url LOG.info('starting download :%s into %s', download_url, targetfile) cmd = 'axel -o %s -n %d -a -s %d %s' % (targetfile, connections, bps, download_url) LOG.debug('running axel: %s', cmd) axel = subprocess.Popen([ 'axel', '-o', targetfile, '-a', '-s', str(bps), '-n', str(connections), download_url ]) currsize = os.path.getsize(targetfile) if os.path.exists(targetfile) else 0 pollinterval = 5 time.sleep(pollinterval) remaining_attempts = 3 while axel.poll() is None: time.sleep(pollinterval) progress = os.path.getsize(targetfile) - currsize currsize = currsize + progress if progress == 0: LOG.warn('seems like axel isnt effective in the last %d seconds', pollinterval) if remaining_attempts == 0: LOG.error('axel seems totally stuck, aborting') axel.kill() return remaining_attempts = remaining_attempts - 1 pollinterval = pollinterval * 2 returncode = axel.poll() if returncode != 0: print '\n[E] Download %s failed!' % download_url LOG.error('download %s failed with code: %d', download_url, returncode) return if os.path.exists(targetfile): if os.path.getsize(targetfile) != filesize: LOG.info('detected partial download %s due to file size', targetfile) try: os.remove(targetfile) except (OSError, IOError): print '\n[E] Cant remove bad download %s' % targetfile LOG.error('cant remove bad download %s', targetfile, exc_info=True)
def __readargs(): try: conf = __getconfig() LOG.info('starting sync in home dir: %s', conf.get('localdir')) print '\n------------------' print "... PutIO/Sync ..." print '\n------------------' print "Minimum Reserved: \t%i" % sync_utils.min_space_to_reserve() print "Available Disk Space: \t%i" % sync_utils.total_free_space() print "Local Sync Dir: \t" + conf.get('localdir') __sync_account(conf) except KeyboardInterrupt: exit_helper.exit_interrupted()
def __readargs(): try: conf = __getconfig() LOG.info('starting sync in home dir: %s', conf.get('localdir')) print '\n------------------' print "... PutIO/Sync ..." print '\n------------------' print "Minimum Reserved: \t%i" % sync_utils.min_space_to_reserve() print "Available Disk Space: \t%i" % sync_utils.total_free_space() print "Local Sync Dir: \t"+conf.get('localdir') __sync_account(conf) except KeyboardInterrupt: exit_helper.exit_interrupted()
def delete_files(root, files): """deletes files and direcotries that exist locally but not in put.io""" for target in files: abspath = os.path.join(root, target) print '\n [!] Deleting %s since its not in the putio account' % abspath LOG.info('deleting %s since its not in the putio account', abspath) if os.path.exists(abspath): try: if os.path.isdir(abspath): shutil.rmtree(abspath) else: os.remove(abspath) except (OSError, ValueError): print '\n [E] Cant delete dir %s' % abspath LOG.error('cant delete dir %s', abspath, exc_info=True) else: LOG.error('wanted to delete %s but it no longer exists', abspath)
def __sync_account(conf): """perfoms the putio sync action :conf: configuration object :returns: None """ print '\n------------------' print '\n...Sync started...' print '\n------------------' localdir = conf.get('localdir') if os.path.exists(localdir) and not os.path.isdir(localdir): exit_helper.exit_cant_sync_locally(localdir) if not os.path.exists(localdir): os.makedirs(localdir) putio_dirtree = {} while True: try: sync_utils.suspend_until_can_store_all(conf) LOG.info('sync loop iteration started') files_to_dirs = {} putio_dirtree = __get_putio_files(conf, 0, putio_dirtree) __create_local_dirs( conf.get('localdir'), putio_dirtree, files_to_dirs) for remoteitem, targetdir in files_to_dirs.iteritems(): fileid = remoteitem.itemid filesize = remoteitem.size download_url = putio_api.get_download_url(conf, fileid) if download_url: sync_utils.start_download( filesize, download_url, targetdir, conf) except Exception: LOG.error('sync iteration failed', exc_info=True) timenow = __gettimenow() print '\n%s :. waiting...' % timenow sync_utils.suspend_sync()
def suspend_until_can_store_all(conf): """ensures local filesystem have enough space on disk to sync :conf: configuration object :returns: """ LOG.info('ensuring enough space in filesystem') while True: localsize = os.path.getsize(conf.get('localdir')) # we don't account for local size because it's replaced if necessary free_space = total_free_space() + localsize putio_size = putio_root_size(conf) if free_space - min_space_to_reserve() < putio_size: print '\n[!] Suspending Sync: not enough space to sync local: %d remote: %d ' % free_space, putio_size LOG.warn('not enough space to sync local: %d remote: %d', free_space, putio_size) suspend_sync() else: break
def delete_files(root, files): """deletes files and direcotries that exist locally but not in put.io""" for target in files: abspath = os.path.join(root, target) print '\n [!] Deleting %s since its not in the putio account' % abspath LOG.info('deleting %s since its not in the putio account', abspath) if os.path.exists(abspath): try: if os.path.isdir(abspath): shutil.rmtree(abspath) else: os.remove(abspath) except (OSError, ValueError): print '\n [E] Cant delete dir %s' % abspath LOG.error('cant delete dir %s', abspath, exc_info=True) else: LOG.error( 'wanted to delete %s but it no longer exists', abspath)
def suspend_until_can_store_all(conf): """ensures local filesystem have enough space on disk to sync :conf: configuration object :returns: """ LOG.info('ensuring enough space in filesystem') while True: localsize = os.path.getsize(conf.get('localdir')) # we don't account for local size because it's replaced if necessary free_space = total_free_space() + localsize putio_size = putio_root_size(conf) if free_space - min_space_to_reserve() < putio_size: print '\n[!] Suspending Sync: not enough space to sync local: %d remote: %d ' % free_space,putio_size LOG.warn('not enough space to sync local: %d remote: %d', free_space, putio_size) suspend_sync() else: break
def __sync_account(conf): """perfoms the putio sync action :conf: configuration object :returns: None """ print '\n------------------' print '\n...Sync started...' print '\n------------------' localdir = conf.get('localdir') if os.path.exists(localdir) and not os.path.isdir(localdir): exit_helper.exit_cant_sync_locally(localdir) if not os.path.exists(localdir): os.makedirs(localdir) putio_dirtree = {} while True: try: sync_utils.suspend_until_can_store_all(conf) LOG.info('sync loop iteration started') files_to_dirs = {} putio_dirtree = __get_putio_files(conf, 0, putio_dirtree) __create_local_dirs(conf.get('localdir'), putio_dirtree, files_to_dirs) for remoteitem, targetdir in files_to_dirs.iteritems(): fileid = remoteitem.itemid filesize = remoteitem.size download_url = putio_api.get_download_url(conf, fileid) if download_url: sync_utils.start_download(filesize, download_url, targetdir, conf) except Exception: LOG.error('sync iteration failed', exc_info=True) timenow = __gettimenow() print '\n%s :. waiting...' % timenow sync_utils.suspend_sync()
def __getconfig(): """creates a config object used later in the script :returns: dictionary with the config """ if not OAUTH_TOKEN: exit_helper.exit_bad_config() oauthtoken = OAUTH_TOKEN if OAUTH_TOKEN_SYMMETRIC_ARMOR_BASE64: LOG.info('app info is encrypted, prompting gpg passphrase') print 'app info is encrypted, running gpg to decrypt' oauthtoken = sync_utils.gpgdecode(oauthtoken) oauthtoken = oauthtoken.strip('\n ') return dict(oauthtoken=oauthtoken, parallel_downloads=PARALLEL_DOWNLOADS, conn_per_downloads=CONNECTIONS_PER_DOWNLOAD, localdir=LOCAL_MIRROR_ROOT, bytes_per_second=MAX_DOWNLOAD_SPEED_BYTES_PER_SECOND)
def __check_filesize_and_crc(targetfile, expected_size, expected_crc32): """check a file for expected size and crc32 :targetfile: file to check :size: expected size :crc: crc32 checksum :returns: True if check is ok """ LOG.info('doing byte count and crc32 check to file %s', targetfile) if os.path.getsize(targetfile) != expected_size: LOG.info('detected partial download %s due to filesize', targetfile) return False else: with open(targetfile, 'r') as binfile: crc32 = binascii.crc32(binfile.read()) & 0xFFFFFFFF crchex = "%08X" % crc32 crchex = crchex.lower() if crchex.encode('utf-8') != expected_crc32.encode('utf-8'): LOG.info( 'detected partial download due to crc32 got: ' + '%s expected: %s file: %s', crchex, expected_crc32, targetfile) return False return True
def __check_filesize_and_crc(targetfile, expected_size, expected_crc32): """check a file for expected size and crc32 :targetfile: file to check :size: expected size :crc: crc32 checksum :returns: True if check is ok """ LOG.info('doing byte count and crc32 check to file %s', targetfile) if os.path.getsize(targetfile) != expected_size: LOG.info( 'detected partial download %s due to filesize', targetfile) return False else: with open(targetfile, 'r') as binfile: crc32 = binascii.crc32(binfile.read()) & 0xFFFFFFFF crchex = "%08X" % crc32 crchex = crchex.lower() if crchex.encode('utf-8') != expected_crc32.encode('utf-8'): LOG.info('detected partial download due to crc32 got: ' + '%s expected: %s file: %s', crchex, expected_crc32, targetfile) return False return True
def start_download(filesize, download_url, targetfile, conf): """downloads the file""" suspend_until_can_store_file(filesize, targetfile) bps = conf.get('bytes_per_second') connections = conf.get('conn_per_downloads') print '\nStarting download :%s' % download_url LOG.info('starting download :%s into %s', download_url, targetfile) cmd = 'axel -o %s -n %d -a -s %d %s' % (targetfile, connections, bps, download_url) LOG.debug('running axel: %s', cmd) axel = subprocess.Popen( ['axel', '-o', targetfile, '-a', '-s', str(bps), '-n', str(connections), download_url]) currsize = os.path.getsize(targetfile) if os.path.exists(targetfile) else 0 pollinterval = 5 time.sleep(pollinterval) remaining_attempts = 3 while axel.poll() is None: time.sleep(pollinterval) progress = os.path.getsize(targetfile) - currsize currsize = currsize + progress if progress == 0: LOG.warn('seems like axel isnt effective in the last %d seconds', pollinterval) if remaining_attempts == 0: LOG.error('axel seems totally stuck, aborting') axel.kill() return remaining_attempts = remaining_attempts - 1 pollinterval = pollinterval * 2 returncode = axel.poll() if returncode != 0: print '\n[E] Download %s failed!' % download_url LOG.error( 'download %s failed with code: %d', download_url, returncode) return if os.path.exists(targetfile): if os.path.getsize(targetfile) != filesize: LOG.info( 'detected partial download %s due to file size', targetfile) try: os.remove(targetfile) except (OSError, IOError): print '\n[E] Cant remove bad download %s' % targetfile LOG.error( 'cant remove bad download %s', targetfile, exc_info=True)