def fetch_prebuilt_wheels(self, binary_base_url, deploy_pants_wheels_path,
deploy_3rdparty_wheels_path, to_dir):
wheel_paths = self.list_prebuilt_wheels(binary_base_url,
deploy_pants_wheels_path,
deploy_3rdparty_wheels_path)
if not wheel_paths:
raise ValueError("No wheels found.")
# Fetching the wheels in parallel
# It is okay to have some interleaving outputs from the fetcher,
# because we are summarizing things in the end.
fetcher = Fetcher(os.getcwd())
checksummer = fetcher.ChecksumListener(digest=hashlib.sha1())
futures = []
with ThreadPoolExecutor(max_workers=8) as executor:
for k in wheel_paths:
file_path, url_path = k.split(self.OUTPUT_DELIMITER)
dest = os.path.join(to_dir, file_path)
safe_mkdir(os.path.dirname(dest))
url = '{}/{}'.format(binary_base_url, url_path)
future = executor.submit(self._download, fetcher, checksummer,
url, dest)
futures.append((future, url))
# Summarize the fetch results.
fail = False
for future, url in futures:
if future.exception() is not None:
logger.error('Failed to download: {}'.format(url))
fail = True
else:
logger.info('Downloaded: {}'.format(url))
if fail:
raise fetcher.Error()
