def extract_multi_async(to_fetch, seen_urls): results = [] for url in to_fetch: if url in seen_urls: continue seen_urls.add(url) try: results.append((yield from extract_async(url))) except Exception: continue return results
def extract_multi_async(to_fetch, seen_urls): results = [] for url in to_fetch: if url in seen_urls: continue seen_urls.add(url) try: results.append((yield from extract_async(url))) except Exception: continue return results
def extract_multi_async(to_fetch, seen_urls): futures, results = [], [] for url in to_fetch: if url in seen_urls: continue seen_urls.add(url) futures.append(extract_async(url)) # Parallel kickoff for future in asyncio.as_completed(futures): # Prioritized wait try: results.append((yield from future)) except Exception: continue return results
def extract_count_async(url, word_length): _, data, found_urls = yield from extract_async(url) top_word = yield from wordcount_async(data, word_length) return url, top_word, found_urls
def extract_count_async(url, word_length): _, data, found_urls = yield from extract_async(url) top_word = yield from wordcount_async(data, word_length) return url, top_word, found_urls