コード例 #1
0
def finance(cur, con, pkey):
	# Wall Street Journal
	articleInfo = ArticleScraper.getTop10Articles("http://www.wsj.com/xml/rss/3_7014.xml", "finance")
	for article in articleInfo:
		if (len(article) < 3):
			continue
		cur.execute(""" INSERT INTO articles (id, source, title, url, summary) VALUES (%s, %s, %s, %s, %s) """, (pkey, "wsj", article[0], article[2], article[1]))
		pkey += 1
	con.commit()

	# New York Times Finance
	articleInfo = ArticleScraper.getTop10Articles("http://rss.nytimes.com/services/xml/rss/nyt/Business.xml", "finance")
	for article in articleInfo:
		if (len(article) < 3):
			continue
		cur.execute(""" INSERT INTO articles (id, source, title, url, summary) VALUES (%s, %s, %s, %s, %s) """, (pkey, "nytf", article[0], article[2], article[1]))
		pkey += 1
	con.commit()

	# Bloomberg
	articleInfo = ArticleScraper.getTop10Articles("http://www.bloomberg.com/markets", "finance")
	for article in articleInfo:
		if (len(article) < 2):
			continue
		cur.execute(""" INSERT INTO articles (id, source, title, url) VALUES (%s, %s, %s, %s) """, (pkey, "bb", article[0], article[1]))
		pkey += 1
	con.commit()

	#BBC Business
	articleInfo = ArticleScraper.getTop10Articles("http://feeds.bbci.co.uk/news/business/rss.xml", "finance")
	for article in articleInfo:
		if (len(article) < 2):
			continue
		cur.execute(""" INSERT INTO articles (id, source, title, url) VALUES (%s, %s, %s, %s) """, (pkey, "bbcb", article[0], article[1]))
		pkey += 1
	con.commit()

	# Reuters
	articleInfo = ArticleScraper.getTop10Articles("http://feeds.reuters.com/reuters/businessNews", "finance")
	for article in articleInfo:
		if (len(article) < 2):
			continue
		cur.execute(""" INSERT INTO articles (id, source, title, url) VALUES (%s, %s, %s, %s) """, (pkey, "rt", article[0], article[1]))
		pkey += 1
	con.commit()
コード例 #2
0
def technology(cur, con, pkey):
	# Hacker News
	articleInfo = ArticleScraper.getTop10Articles("http://news.ycombinator.com", "technology")
	for article in articleInfo:
		if (len(article) < 5):
			continue
		cur.execute(""" INSERT INTO articles (id, source, title, url, comment_url, comment_number, score) VALUES (%s, %s, %s, %s, %s, %s, %s) """, (pkey, "hn", article[0], article[1], article[4], article[3], article[2]))
		pkey += 1
	con.commit()

    # New York Times Tech
	articleInfo = ArticleScraper.getTop10Articles("http://rss.nytimes.com/services/xml/rss/nyt/Technology.xml", "technology")
	for article in articleInfo:
		if (len(article) < 3):
			continue
		cur.execute(""" INSERT INTO articles (id, source, title, url, summary) VALUES (%s, %s, %s, %s, %s) """, (pkey, "nyt", article[0], article[2], article[1]))
		pkey += 1
	con.commit()

	# TechCrunch
	articleInfo = ArticleScraper.getTop10Articles("http://www.techcrunch.com", "technology")
	for article in articleInfo:
		if (len(article) < 2):
			continue
		cur.execute(""" INSERT INTO articles (id, source, title, url) VALUES (%s, %s, %s, %s) """, (pkey, "tc", article[0], article[1]))
		pkey += 1
	con.commit()

	# Recode
	articleInfo = ArticleScraper.getTop10Articles("http://www.recode.net", "technology")
	for article in articleInfo:
		if (len(article) < 2):
			continue
		cur.execute(""" INSERT INTO articles (id, source, title, url) VALUES (%s, %s, %s, %s) """, (pkey, "rc", article[0], article[1]))
		pkey += 1
	con.commit()

	return pkey