class TestFetcher(unittest.TestCase): "test case for Fetcher" def setUp(self): self.fetcher = FeedFetcher() def tearDown(self): pass def test_fetch(self): url = "./t/testdata/test.rss" entries = self.fetcher.fetch(url) self.assertEqual(len(entries), 10) self.assertEqual(entries[0].url, "http://hylom.net/2012/08/08/emacs-24-on-mac-os-x/")
def main(): db = MySQLDatabase() fetcher = FeedFetcher() feeds = db.get_feeds(offset=0, limit=10) read_count = 10 while len(feeds) > 0: for feed in feeds: fid = feed[0] url = feed[1] title = feed[2] print "fetching #{0}: {1}".format(fid, url) entries = fetcher.fetch(url) for entry in entries: entry.feed_id = fid try: print "insert {0}".format(entry.url) except UnicodeEncodeError: print "insert {0}".format(entry.url.encode('utf-8')) db.append_feed_content(entry) feeds = db.get_feeds(offset=read_count, limit=10) read_count += 10