def fetch(self,
urls: List[str]) -> Generator[List[FetchResult], None, None]:
"""Fetches urls parallel in batches and returns a generator that yields every fetched URL batch as a list of FetchResult objects."""
loop = self._get_event_loop()
queue = URLQueue(urls=urls)
while not queue.is_empty():
url_batch = [
next(queue) for _ in range(self.fetch_batch_size)
if not queue.is_empty()
]
yield loop.run_until_complete(self._fetch_parallel_job(url_batch))
def test_url_queue_is_empty_false(self, url_queue: URLQueue):
url_queue.add(self.test_urls)
assert not url_queue.is_empty()
def test_process_url_queue(self, url_queue: URLQueue):
url_queue.add(self.test_urls)
for url in url_queue:
assert url in self.test_urls
assert url_queue.is_empty()
def test_url_queue_is_empty_true(self, url_queue: URLQueue):
assert url_queue.is_empty()