class IssuesScraper(Scraper): def __init__(self): Scraper.__init__(self) self.url = "https://berniesanders.com/issues/feed/" self.html = HTMLParser() self.issue_provider = IssueProvider() self.push_provider = PushProvider() def collect_urls(self): records = [] items = self.get(self.url).findAll("item") for item in items: record = { "title": self.html.unescape(item.title.text), "timestamp_publish": parser.parse(item.pubdate.text), "site": "berniesanders.com", "lang": "en", "description_html": item.description.text, "description": self.html.unescape( BeautifulSoup(item.description.text).p.text), "url": item.link.nextSibling } records.append(record) return records def retrieve(self, record): soup = self.get(record["url"]) # retrieve image from <meta property="og:image" content="https://berniesanders.com/wp-content/uploads/2015/07/072615_Bernie_NewOrleans-4382.jpg"/> meta_image = soup.findAll(attrs={"property":"og:image"}) record["image_url"] = meta_image[0]["content"].encode('utf8') # reset soup to content soup = self.sanitize_soup(soup.find("section", {"id": "content"})) while soup.article.style is not None: soup.article.style.extract() record["body_html"] = str(soup.article) text = [] for elem in soup.article.recursiveChildGenerator(): if isinstance(elem, types.StringTypes): text.append(self.html.unescape(elem.strip())) elif elem.name == 'br': text.append("") record["body"] = "\n".join(text) record['body_markdown'] = convert_markdown (record['body_html']) return record def go(self): urls = self.collect_urls() if not urls: logging.critical("Could not retrieve issues.") sys.exit(1) for url in urls: record = self.retrieve(url) if self.issue_provider.exists_by_url(record["url"]): print "found" else: msg = "Inserting record for '{0}'." logging.info(msg.format(record["title"].encode("utf8"))) record["timestamp_creation"] = datetime.now() result = self.issue_provider.create(record) self.push_provider.create_by_foreign_model(result)
class ArticlesScraper(Scraper): def __init__(self): Scraper.__init__(self) self.url = "https://berniesanders.com/daily/" self.html = HTMLParser() self.article_provider = ArticleProvider() self.push_provider = PushProvider() def retrieve_article(self, url): for x in range(3): r = requests.get(url) if "https://berniesanders.com" not in r.url: return r.url, False, False if r.status_code == 200: soup = BeautifulSoup(r.text) soup = self.sanitize_soup(soup) image = soup.find('meta', {'property': 'og:image'})['content'] content = soup.article paragraphs = [ self.html.unescape(self.replace_with_newlines(p)) for p in content.findAll("p") ] text = "\n\n".join(paragraphs) html = "".join([str(p) for p in content.findAll("p")]) return text, html, image return False, False, False def go(self): soup = self.get(self.url) content = soup.find("section", {"id": "content"}) for article in content.findAll("article"): rec = { "article_id": article['id'], "image_url": "", "body": "", "timestamp_publish": self.choose_publish_date(article.time["datetime"]), "site": "berniesanders.com", "lang": "en", "article_type": "DemocracyDaily", "excerpt": self.html.unescape( article.find("div", { "class": "excerpt" }).p.text), "title": self.html.unescape(article.h2.text), "article_category": self.html.unescape(article.h1.string.strip()), "url": article.h2.a["href"] } if article.img is not None: rec["image_url"] = article.img["src"] text, html, image = self.retrieve_article(rec["url"]) if text and not html: rec["body"] = text rec["body_markdown"] = text rec['article_type'] = "ExternalLink" elif text and html: rec["body"] = text rec['body_markdown'] = convert_markdown(html) exit(0) try: article["image_url"] except KeyError: article["image_url"] = image msg = "" if self.article_provider.exists_by_article_id(rec["article_id"]): print "found" else: print "not found" msg = "Inserting '{0}', created {1}" result = self.article_provider.create(rec) self.push_provider.create_by_foreign_model(result) logging.info( msg.format(rec["title"].encode("utf8"), str(rec["timestamp_publish"])))
class NewsScraper(Scraper): def __init__(self): Scraper.__init__(self) self.url = "https://berniesanders.com/news/" self.html = HTMLParser() self.news_provider = NewsProvider() self.push_provider = PushProvider() def retrieve_article(self, url): for x in range(3): r = requests.get(url) if "https://berniesanders.com" not in r.url: return r.url, False, False if r.status_code == 200: soup = BeautifulSoup(r.text) soup = self.sanitize_soup(soup) image = soup.find('meta', {'property': 'og:image'})['content'] content = soup.article paragraphs = [self.html.unescape(self.replace_with_newlines(p)) for p in content.findAll("p")] text = "\n\n".join(paragraphs) html = "".join([str(p) for p in content.findAll("p")]) return text, html, image return False, False, False def go(self): soup = self.get(self.url) content = soup.find("section", {"id": "content"}) for article in content.findAll("article"): rec = { "news_id": article['id'], "image_url": "", "timestamp_publish": parser.parse(article.time["datetime"]), "site": "berniesanders.com", "lang": "en", "title": self.html.unescape(article.h2.text), "news_category": self.html.unescape(article.h1.string.strip()), "url": article.h2.a["href"] } if article.img is not None: rec["image_url"] = article.img["src"] # Pull excerpt if available try: rec["excerpt_html"] = str(article.p) rec["excerpt"] = self.html.unescape(article.p.text) except AttributeError: rec["excerpt"], rec["excerpt_html"] = "", "" # Determine Type if rec['news_category'].lower() in ["on the road", "news"]: rec['news_type'] = "News" elif rec['news_category'].lower() == "press release": rec['news_type'] = "PressRelease" else: rec['news_type'] = "Unknown" text, html, image = self.retrieve_article(rec["url"]) if text and not html: rec["body"], rec["body_html"] = text, text rec['news_type'] = "ExternalLink" rec["body_html_nostyle"] = "" elif text and html: rec["body"], rec["body_html"] = text, html no_style = self.remove_style(BeautifulSoup(html)) rec["body_html_nostyle"] = "".join([str(p) for p in no_style.findAll("p")]) try: article["image_url"] except KeyError: article["image_url"] = image rec['body_markdown'] = convert_markdown (rec['body_html']) msg = "" if self.news_provider.exists_by_news_id(rec["news_id"]): print "found" else: print "not found" msg = "Inserting '{0}', created {1}" result = self.news_provider.create(rec) self.push_provider.create_by_foreign_model(result) logging.info(msg.format( rec["title"].encode("utf8"), str(rec["timestamp_publish"]) ))
class ArticlesScraper(Scraper): def __init__(self): Scraper.__init__(self) self.url = "https://berniesanders.com/daily/" self.html = HTMLParser() self.article_provider = ArticleProvider() self.push_provider = PushProvider() def retrieve_article(self, url): for x in range(3): r = requests.get(url) if "https://berniesanders.com" not in r.url: return r.url, False, False if r.status_code == 200: soup = BeautifulSoup(r.text) soup = self.sanitize_soup(soup) image = soup.find('meta', {'property': 'og:image'})['content'] content = soup.article paragraphs = [self.html.unescape(self.replace_with_newlines(p)) for p in content.findAll("p")] text = "\n\n".join(paragraphs) html = "".join([str(p) for p in content.findAll("p")]) return text, html, image return False, False, False def go(self): soup = self.get(self.url) content = soup.find("section", {"id": "content"}) for article in content.findAll("article"): rec = { "article_id": article['id'], "image_url": "", "body": "", "timestamp_publish": self.choose_publish_date(article.time["datetime"]), "site": "berniesanders.com", "lang": "en", "article_type": "DemocracyDaily", "excerpt": self.html.unescape( article.find( "div", {"class": "excerpt"}).p.text), "title": self.html.unescape(article.h2.text), "article_category": self.html.unescape(article.h1.string.strip()), "url": article.h2.a["href"] } if article.img is not None: rec["image_url"] = article.img["src"] text, html, image = self.retrieve_article(rec["url"]) if text and not html: rec["body"] = text rec["body_markdown"] = text rec['article_type'] = "ExternalLink" elif text and html: rec["body"] = text rec['body_markdown'] = convert_markdown (html) exit(0) try: article["image_url"] except KeyError: article["image_url"] = image msg = "" if self.article_provider.exists_by_article_id(rec["article_id"]): print "found" else: print "not found" msg = "Inserting '{0}', created {1}" result = self.article_provider.create(rec) self.push_provider.create_by_foreign_model(result) logging.info(msg.format( rec["title"].encode("utf8"), str(rec["timestamp_publish"]) ))
class NewsScraper(Scraper): def __init__(self, url): Scraper.__init__(self) self.url = url self.html = HTMLParser() self.news_provider = NewsProvider() self.push_provider = PushProvider() def retrieve_article(self, url): for x in range(3): r = requests.get(url) if "https://berniesanders.com" not in r.url: return r.url, False, False if r.status_code == 200: soup = BeautifulSoup(r.text) soup = self.sanitize_soup(soup) image = soup.find('meta', {'property': 'og:image'})['content'] content = soup.article paragraphs = [ self.html.unescape(self.replace_with_newlines(p)) for p in content.findAll("p") ] text = "\n\n".join(paragraphs) html = "".join([str(p) for p in content.findAll("p")]) return text, html, image return False, False, False def go(self): soup = self.get(self.url) try: lang = soup.html['lang'] except KeyError as e: lang = 'en' content = soup.find("section", {"id": "content"}) for article in content.findAll("article"): rec = { "news_id": article['id'], "body": "", "image_url": "", "timestamp_publish": self.choose_publish_date(article.time["datetime"]), "site": "berniesanders.com", "lang": lang, "title": self.html.unescape(article.h2.text), "news_category": self.html.unescape(article.h1.string.strip()), "url": article.h2.a["href"] } if article.img is not None: rec["image_url"] = article.img["src"] # Pull excerpt if available try: rec["excerpt"] = self.html.unescape(article.p.text) except AttributeError: rec["excerpt"] = "" # Determine Type if rec['news_category'].lower() in ["on the road", "news"]: rec['news_type'] = "News" elif rec['news_category'].lower() in [ "press release", "comunicados de prensa" ]: rec['news_type'] = "PressRelease" else: rec['news_type'] = "Unknown" text, html, image = self.retrieve_article(rec["url"]) if text and not html: rec["body"], rec["body_markdown"] = text, text rec['news_type'] = "ExternalLink" elif text and html: rec["body"] = text rec['body_markdown'] = convert_markdown(html) try: article["image_url"] except KeyError: article["image_url"] = image msg = "" if self.news_provider.exists_by_news_id(rec["news_id"]): print "found" else: print "not found" msg = "Inserting '{0}', created {1}" result = self.news_provider.create(rec) self.push_provider.create_by_foreign_model(result) logging.info( msg.format(rec["title"].encode("utf8"), str(rec["timestamp_publish"])))