def _load_all_urls(collections): """Pass in a list of Fetcher objects and this will load all the urls needed and populate the objects.""" needed = [] for c in collections: needed.append(c.url) to_load = set(needed) url_content = multihttp.request_urls(to_load) for c in collections: c.content, ex = url_content[c.url]
def populate_articles(self, items): to_fetch = [host + x['path'] for x in items] fetched = multihttp.request_urls(to_fetch) articles = [] for item in items: try: content, ex = fetched[host + item['path']] if ex is not None: raise ex a = parse_article(content) if a is not None: a['url'] = host + item['path'] a['teaser'] = item['teaser'] a['path'] = item['path'] articles.append(a) except Exception, e: print e