class TestFetcher(unittest.TestCase):
"test case for Fetcher"
def setUp(self):
self.fetcher = FeedFetcher()
def tearDown(self):
pass
def test_fetch(self):
url = "./t/testdata/test.rss"
entries = self.fetcher.fetch(url)
self.assertEqual(len(entries), 10)
self.assertEqual(entries[0].url, "http://hylom.net/2012/08/08/emacs-24-on-mac-os-x/")
def main():
db = MySQLDatabase()
fetcher = FeedFetcher()
feeds = db.get_feeds(offset=0, limit=10)
read_count = 10
while len(feeds) > 0:
for feed in feeds:
fid = feed[0]
url = feed[1]
title = feed[2]
print "fetching #{0}: {1}".format(fid, url)
entries = fetcher.fetch(url)
for entry in entries:
entry.feed_id = fid
try:
print "insert {0}".format(entry.url)
except UnicodeEncodeError:
print "insert {0}".format(entry.url.encode('utf-8'))
db.append_feed_content(entry)
feeds = db.get_feeds(offset=read_count, limit=10)
read_count += 10