async def _fetch_galaxy_pages(self): """ Fetch the roles in a remote repository. Returns: async generator: dicts that represent pages from galaxy api """ page_count = 0 remote = self.remote progress_data = dict(message="Parsing Pages from Galaxy Roles API", code="parsing.roles") with ProgressReport(**progress_data) as progress_bar: api_version = get_api_version(remote.url) downloader = remote.get_downloader(url=get_page_url(remote.url, api_version)) metadata = parse_metadata(await downloader.run()) page_count = math.ceil(float(metadata["count"]) / float(PAGE_SIZE)) progress_bar.total = page_count progress_bar.save() yield metadata progress_bar.increment() # Concurrent downloads are limited by aiohttp... not_done = set( remote.get_downloader(url=get_page_url(remote.url, api_version, page)).run() for page in range(2, page_count + 1) ) while not_done: done, not_done = await asyncio.wait(not_done, return_when=FIRST_COMPLETED) for item in done: yield parse_metadata(item.result()) progress_bar.increment()
async def _get_url(page, versions_url=None): if collection_info and not versions_url: name, version, source = collection_info[page - 1] namespace, name = name.split(".") root = source or remote.url api_endpoint = (await _get_collection_api(root))[0] url = f"{api_endpoint}{namespace}/{name}/" return url if not versions_url: api_endpoint, api_version = await _get_collection_api(remote.url) return get_page_url(api_endpoint, api_version, page) if not self.api_version: await _get_collection_api(remote.url) return get_page_url(versions_url, self.api_version, page)
def _get_url(page): if collection_info: name, version, source = collection_info[page - 1] namespace, name = name.split(".") root = source or remote.url url = f"{root}/api/v2/collections/{namespace}/{name}" return url return get_page_url(remote.url, page)
def _get_url(page, api_version): if collection_info: name, version, source = collection_info[page - 1] namespace, name = name.split(".") root = source or remote.url url = f"{root.rstrip('/')}/{namespace}/{name}" return url return get_page_url(remote.url, api_version, page)