def download_file(self, embeddings_folder, logger): logger.info('Downloading word embeddings file for "{}"...'.format( self.alias)) out_path = os.path.join(embeddings_folder, self.embeddings_filename) if os.path.exists(out_path): logger.info('Already downloaded -> skipping!') else: url = '{}/{}'.format(self.base_url, self.embeddings_filename) r = requests.get(url, stream=True) total_size = int( r.headers.get('content-length', self.approximate_filesize) ) # size of the embeddings file (bytes) chunk_size = 4 * 1024 * 1024 # 4 MB bar = DataTransferBar(max_value=total_size).start() completed_bytes = 0 try: with open(out_path, 'wb') as f: for chunk in r.iter_content(chunk_size=chunk_size): if chunk: f.write(chunk) completed_bytes += len(chunk) if completed_bytes > bar.max_value: bar.max_value = completed_bytes bar.update(completed_bytes) except BaseException as e: os.unlink(out_path) raise e bar.finish() logger.info('Done!')
def progress_urlretrieve(url, ofile): print("Downloading %s ..." % url) sys.stdout.flush() try: from progressbar import DataTransferBar, UnknownLength pbar = DataTransferBar() def _upd(count, size, total): if pbar.max_value is None: if total > 0: pbar.start(total) else: pbar.start(UnknownLength) pbar.update(min(count * size, total)) sys.stdout.flush() res = _urlretrieve(url, ofile, reporthook=_upd) try: pbar.finish() except: pass return res except ImportError: return _urlretrieve(url, ofile)
def test_known_length(): dtb = DataTransferBar().start(max_value=50) for i in range(50): dtb.update(i) dtb.finish()
def test_unknown_length(): dtb = DataTransferBar().start(max_value=progressbar.UnknownLength) for i in range(50): dtb.update(i) dtb.finish()