def finance(cur, con, pkey): # Wall Street Journal articleInfo = ArticleScraper.getTop10Articles("http://www.wsj.com/xml/rss/3_7014.xml", "finance") for article in articleInfo: if (len(article) < 3): continue cur.execute(""" INSERT INTO articles (id, source, title, url, summary) VALUES (%s, %s, %s, %s, %s) """, (pkey, "wsj", article[0], article[2], article[1])) pkey += 1 con.commit() # New York Times Finance articleInfo = ArticleScraper.getTop10Articles("http://rss.nytimes.com/services/xml/rss/nyt/Business.xml", "finance") for article in articleInfo: if (len(article) < 3): continue cur.execute(""" INSERT INTO articles (id, source, title, url, summary) VALUES (%s, %s, %s, %s, %s) """, (pkey, "nytf", article[0], article[2], article[1])) pkey += 1 con.commit() # Bloomberg articleInfo = ArticleScraper.getTop10Articles("http://www.bloomberg.com/markets", "finance") for article in articleInfo: if (len(article) < 2): continue cur.execute(""" INSERT INTO articles (id, source, title, url) VALUES (%s, %s, %s, %s) """, (pkey, "bb", article[0], article[1])) pkey += 1 con.commit() #BBC Business articleInfo = ArticleScraper.getTop10Articles("http://feeds.bbci.co.uk/news/business/rss.xml", "finance") for article in articleInfo: if (len(article) < 2): continue cur.execute(""" INSERT INTO articles (id, source, title, url) VALUES (%s, %s, %s, %s) """, (pkey, "bbcb", article[0], article[1])) pkey += 1 con.commit() # Reuters articleInfo = ArticleScraper.getTop10Articles("http://feeds.reuters.com/reuters/businessNews", "finance") for article in articleInfo: if (len(article) < 2): continue cur.execute(""" INSERT INTO articles (id, source, title, url) VALUES (%s, %s, %s, %s) """, (pkey, "rt", article[0], article[1])) pkey += 1 con.commit()
def technology(cur, con, pkey): # Hacker News articleInfo = ArticleScraper.getTop10Articles("http://news.ycombinator.com", "technology") for article in articleInfo: if (len(article) < 5): continue cur.execute(""" INSERT INTO articles (id, source, title, url, comment_url, comment_number, score) VALUES (%s, %s, %s, %s, %s, %s, %s) """, (pkey, "hn", article[0], article[1], article[4], article[3], article[2])) pkey += 1 con.commit() # New York Times Tech articleInfo = ArticleScraper.getTop10Articles("http://rss.nytimes.com/services/xml/rss/nyt/Technology.xml", "technology") for article in articleInfo: if (len(article) < 3): continue cur.execute(""" INSERT INTO articles (id, source, title, url, summary) VALUES (%s, %s, %s, %s, %s) """, (pkey, "nyt", article[0], article[2], article[1])) pkey += 1 con.commit() # TechCrunch articleInfo = ArticleScraper.getTop10Articles("http://www.techcrunch.com", "technology") for article in articleInfo: if (len(article) < 2): continue cur.execute(""" INSERT INTO articles (id, source, title, url) VALUES (%s, %s, %s, %s) """, (pkey, "tc", article[0], article[1])) pkey += 1 con.commit() # Recode articleInfo = ArticleScraper.getTop10Articles("http://www.recode.net", "technology") for article in articleInfo: if (len(article) < 2): continue cur.execute(""" INSERT INTO articles (id, source, title, url) VALUES (%s, %s, %s, %s) """, (pkey, "rc", article[0], article[1])) pkey += 1 con.commit() return pkey