def main(): db = MySQLDatabase() fetcher = FeedFetcher() feeds = db.get_feeds(offset=0, limit=10) read_count = 10 while len(feeds) > 0: for feed in feeds: fid = feed[0] url = feed[1] title = feed[2] print "fetching #{0}: {1}".format(fid, url) entries = fetcher.fetch(url) for entry in entries: entry.feed_id = fid try: print "insert {0}".format(entry.url) except UnicodeEncodeError: print "insert {0}".format(entry.url.encode('utf-8')) db.append_feed_content(entry) feeds = db.get_feeds(offset=read_count, limit=10) read_count += 10
class TestMySQLDatabase(unittest.TestCase): "test case for MySQLDatabase" def setUp(self): self.db = MySQLDatabase() self.db.config["database"] = "testdb" # self.db.create_tables() def tearDown(self): self.db.delete_all() # self.db.delete_tables() def test_append_feed_contents(self): self.db.append_feed_content(TESTDATA01) def test_append_and_get_feed_contents(self): feed_id = 0 self.db.append_feed_content(TESTDATA02) contents = self.db.get_feed_contents(feed_id) urls = [x.url for x in contents] self.assertIn(TESTDATA02.url, urls) self.db.append_feed_content(TESTDATA12) contents = self.db.get_feed_contents(feed_id) self.assertIn(TESTDATA12.url, urls) titles = [x.title for x in contents] self.assertIn(TESTDATA12.title, titles) def test_append_feed(self): url = "http://example.com/" title = "test-title" self.db.append_feed(url, title) def test_append_and_get_feed(self): url = "http://example.com/test01.rss" title = "test-title01" self.db.append_feed(url, title) feeds = self.db.get_feeds() urls = [x[1] for x in feeds] self.assertIn(url, urls) titles = [x[2] for x in feeds] self.assertIn(title, titles)