def _load_all_urls(collections):
"""Pass in a list of Fetcher objects and this will load all the urls needed and populate the objects."""
needed = []
for c in collections:
needed.append(c.url)
to_load = set(needed)
url_content = multihttp.request_urls(to_load)
for c in collections:
c.content, ex = url_content[c.url]
def populate_articles(self, items):
to_fetch = [host + x['path'] for x in items]
fetched = multihttp.request_urls(to_fetch)
articles = []
for item in items:
try:
content, ex = fetched[host + item['path']]
if ex is not None:
raise ex
a = parse_article(content)
if a is not None:
a['url'] = host + item['path']
a['teaser'] = item['teaser']
a['path'] = item['path']
articles.append(a)
except Exception, e:
print e
