def download_file(self, embeddings_folder, logger): logger.info('Downloading word embeddings file for "{}"...'.format( self.alias)) out_path = os.path.join(embeddings_folder, self.embeddings_filename) if os.path.exists(out_path): logger.info('Already downloaded -> skipping!') else: url = '{}/{}'.format(self.base_url, self.embeddings_filename) r = requests.get(url, stream=True) total_size = int( r.headers.get('content-length', self.approximate_filesize) ) # size of the embeddings file (bytes) chunk_size = 4 * 1024 * 1024 # 4 MB bar = DataTransferBar(max_value=total_size).start() completed_bytes = 0 try: with open(out_path, 'wb') as f: for chunk in r.iter_content(chunk_size=chunk_size): if chunk: f.write(chunk) completed_bytes += len(chunk) if completed_bytes > bar.max_value: bar.max_value = completed_bytes bar.update(completed_bytes) except BaseException as e: os.unlink(out_path) raise e bar.finish() logger.info('Done!')
def test_known_length(): dtb = DataTransferBar().start(max_value=50) for i in range(50): dtb.update(i) dtb.finish()
def test_unknown_length(): dtb = DataTransferBar().start(max_value=progressbar.UnknownLength) for i in range(50): dtb.update(i) dtb.finish()