async def _fetch_galaxy_pages(self):
"""
Fetch the roles in a remote repository.
Returns:
async generator: dicts that represent pages from galaxy api
"""
page_count = 0
remote = self.remote
progress_data = dict(message="Parsing Pages from Galaxy Roles API", code="parsing.roles")
with ProgressReport(**progress_data) as progress_bar:
api_version = get_api_version(remote.url)
downloader = remote.get_downloader(url=get_page_url(remote.url, api_version))
metadata = parse_metadata(await downloader.run())
page_count = math.ceil(float(metadata["count"]) / float(PAGE_SIZE))
progress_bar.total = page_count
progress_bar.save()
yield metadata
progress_bar.increment()
# Concurrent downloads are limited by aiohttp...
not_done = set(
remote.get_downloader(url=get_page_url(remote.url, api_version, page)).run()
for page in range(2, page_count + 1)
)
while not_done:
done, not_done = await asyncio.wait(not_done, return_when=FIRST_COMPLETED)
for item in done:
yield parse_metadata(item.result())
progress_bar.increment()
async def _get_url(page, versions_url=None):
if collection_info and not versions_url:
name, version, source = collection_info[page - 1]
namespace, name = name.split(".")
root = source or remote.url
api_endpoint = (await _get_collection_api(root))[0]
url = f"{api_endpoint}{namespace}/{name}/"
return url
if not versions_url:
api_endpoint, api_version = await _get_collection_api(remote.url)
return get_page_url(api_endpoint, api_version, page)
if not self.api_version:
await _get_collection_api(remote.url)
return get_page_url(versions_url, self.api_version, page)
def _get_url(page):
if collection_info:
name, version, source = collection_info[page - 1]
namespace, name = name.split(".")
root = source or remote.url
url = f"{root}/api/v2/collections/{namespace}/{name}"
return url
return get_page_url(remote.url, page)
def _get_url(page, api_version):
if collection_info:
name, version, source = collection_info[page - 1]
namespace, name = name.split(".")
root = source or remote.url
url = f"{root.rstrip('/')}/{namespace}/{name}"
return url
return get_page_url(remote.url, api_version, page)
