def extract_multi_async(to_fetch, seen_urls):
results = []
for url in to_fetch:
if url in seen_urls: continue
seen_urls.add(url)
try:
results.append((yield from extract_async(url)))
except Exception:
continue
return results
def extract_multi_async(to_fetch, seen_urls):
results = []
for url in to_fetch:
if url in seen_urls: continue
seen_urls.add(url)
try:
results.append((yield from extract_async(url)))
except Exception:
continue
return results
def extract_multi_async(to_fetch, seen_urls):
futures, results = [], []
for url in to_fetch:
if url in seen_urls: continue
seen_urls.add(url)
futures.append(extract_async(url)) # Parallel kickoff
for future in asyncio.as_completed(futures): # Prioritized wait
try:
results.append((yield from future))
except Exception:
continue
return results
def extract_count_async(url, word_length):
_, data, found_urls = yield from extract_async(url)
top_word = yield from wordcount_async(data, word_length)
return url, top_word, found_urls
def extract_count_async(url, word_length):
_, data, found_urls = yield from extract_async(url)
top_word = yield from wordcount_async(data, word_length)
return url, top_word, found_urls
