def _prepare_download( resp, # type: Response link, # type: Link progress_bar # type: str ): # type: (...) -> Iterable[bytes] total_length = _get_http_response_size(resp) if link.netloc == PyPI.file_storage_domain: url = link.show_url else: url = link.url_without_fragment logged_url = redact_auth_from_url(url) if total_length: logged_url = '{} ({})'.format(logged_url, format_size(total_length)) if is_from_cache(resp): logger.info("Using cached %s", logged_url) else: logger.info("Downloading %s", logged_url) if logger.getEffectiveLevel() > logging.INFO: show_progress = False elif is_from_cache(resp): show_progress = False elif not total_length: show_progress = True elif total_length > (40 * 1000): show_progress = True else: show_progress = False chunks = response_chunks(resp, CONTENT_CHUNK_SIZE) if not show_progress: return chunks return DownloadProgressProvider(progress_bar, max=total_length)(chunks)
def _download_url( resp, # type: Response link, # type: Link content_file, # type: IO[Any] hashes, # type: Optional[Hashes] progress_bar # type: str ): # type: (...) -> None try: total_length = int(resp.headers['content-length']) except (ValueError, KeyError, TypeError): total_length = 0 if link.netloc == PyPI.file_storage_domain: url = link.show_url else: url = link.url_without_fragment redacted_url = redact_auth_from_url(url) if total_length: logger.info("Downloading %s (%s)", redacted_url, format_size(total_length)) elif is_from_cache(resp): logger.info("Using cached %s", redacted_url) else: logger.info("Downloading %s", redacted_url) if logger.getEffectiveLevel() > logging.INFO: show_progress = False elif is_from_cache(resp): show_progress = False elif total_length > (40 * 1000): show_progress = True elif not total_length: show_progress = True else: show_progress = False def written_chunks(chunks): for chunk in chunks: content_file.write(chunk) yield chunk progress_indicator = _progress_indicator if show_progress: # We don't show progress on cached responses progress_indicator = DownloadProgressProvider(progress_bar, max=total_length) downloaded_chunks = written_chunks( progress_indicator(response_chunks(resp, CONTENT_CHUNK_SIZE), CONTENT_CHUNK_SIZE)) if hashes: hashes.check_against_chunks(downloaded_chunks) else: consume(downloaded_chunks)
def _prepare_download( resp: Response, link: Link, progress_bar: str, ) -> Iterable[bytes]: total_length = _get_http_response_size(resp) if link.netloc == PyPI.file_storage_domain: url = link.show_url else: url = link.url_without_fragment logged_url = redact_auth_from_url(url) if total_length: logged_url = "{} ({})".format(logged_url, format_size(total_length)) if is_from_cache(resp): logger.info("Using cached %s", logged_url) else: logger.info("Downloading %s", logged_url) if logger.getEffectiveLevel() > logging.INFO: show_progress = False elif is_from_cache(resp): show_progress = False elif not total_length: show_progress = True elif total_length > (40 * 1000): show_progress = True else: show_progress = False chunks = response_chunks(resp, CONTENT_CHUNK_SIZE) if not show_progress: return chunks renderer = get_download_progress_renderer(bar_type=progress_bar, size=total_length) return renderer(chunks)
progress_bar # type: str ): # type: (...) -> Iterable[bytes] total_length = _get_http_response_size(resp) if link.netloc == PyPI.file_storage_domain: url = link.show_url else: url = link.url_without_fragment logged_url = redact_auth_from_url(url) if total_length: logged_url = '{} ({})'.format(logged_url, format_size(total_length)) if is_from_cache(resp): logger.info("Using cached %s", logged_url) else: logger.info("Downloading %s", logged_url) if logger.getEffectiveLevel() > logging.INFO: show_progress = False elif is_from_cache(resp): show_progress = False elif not total_length: show_progress = True elif total_length > (40 * 1000): show_progress = True else: show_progress = False