class TestHub:
def setUp(self):
self.hub = MokshaHub()
def tearDown(self):
self.hub.close()
def test_hub_creation(self):
assert_true(self.hub)
eq_(self.hub.topics, {})
class TestHub:
def setUp(self):
self.hub = MokshaHub()
def tearDown(self):
self.hub.close()
def test_hub_creation(self):
assert_true(self.hub)
eq_(self.hub.topics, {})
def __init__(self):
self.parsed = 1
self.hub = MokshaHub()
self.post_processors = []
for entry in pkg_resources.iter_entry_points(
'moksha.feeds.post_processor'):
log.info('Registering feed post-processor: %s' % entry.name)
self.post_processors.append(entry.load())
def __init__(self):
self.parsed = 1
self.hub = MokshaHub()
class FeederProtocol(object):
max_age = int(config.get('feed.max_age', 300))
timeout = int(config.get('feed.timeout', 30))
def __init__(self):
self.parsed = 1
self.hub = MokshaHub()
def is_cached(self, site):
already_got = feed_storage.get(site)
if already_got:
elapsed_time = time.time() - already_got[0]
if elapsed_time < self.max_age:
return True
else:
return False
else:
return False
def on_error(self, traceback, extra_args):
log.error(extra_args)
log.exception(traceback)
def get_feed_from_cache(self, data, key=None):
""" Return feed data from the cache based on a given ``key`` """
log.debug('Getting cached feed for %s' % key)
return defer.succeed(feed_storage.get(key, key)[1])
def parse_feed(self, feed, url):
if not feed:
log.warning('parse_feed got %r for %s' % (feed, url))
return {}
if not isinstance(feed, basestring):
feed = str(feed)
feed = feedparser.parse(StringIO(feed))
assert feed
if feed.get('bozo_exception'):
bozo_exc = str(feed['bozo_exception'])
log.warning("Feed %s getting bozo_exception %r" % (feed, bozo_exc))
feed['bozo_exception'] = bozo_exc
return feed
def store_feed(self, feed, addr):
feed_storage[addr] = (time.time(), feed)
return feed
def get_feed(self, addr):
try:
return feed_storage[addr][1]
except KeyError:
return None
def process_feed(self, parsed_feed, addr, olddata):
""" Process the parsed feed.
If `olddata` is provided, this method will look for new feed entries,
and send notifications to the `feed.$FEED_URL` MokshaHub Topic.
:param parsed_feed: A parsed :mod:`feedcache` feed
:param addr: The URL of the feed
:param olddata: The cached feed data
"""
if not parsed_feed:
log.error("Cannot process %r feed for %s" % (parsed_feed, addr))
return
chan = parsed_feed.get('channel', None)
if chan:
log.debug(chan.get('title', ''))
# Previous data provided; look for new entries.
if olddata:
oldtitles = [entry.get('title') for entry in olddata['entries']]
new_entries = parsed_feed.get('entries', [{}])
if not len(new_entries):
log.warning('Feed contains empty entries: %s' % addr)
return
# If there are no new entries, move on...
newtitle = new_entries[0].get('title', None)
if newtitle == oldtitles[0]:
return
# Send notifications for each new entry
for entry in new_entries[::-1]:
entry_title = entry.get('title', '[No Title]')
channel_link = entry.get('channel', {'link': addr})['link']
if entry['title'] not in oldtitles:
log.info('New feed entry found: %s' % entry['title'])
self.hub.send_message(['feed.%s' % channel_link, 'feeds'],
{'title': entry_title, 'link': entry.get('link')})
def get_page(self, data, args):
return conditional_get_page(args, timeout=self.timeout)
def start(self, data=None):
d = defer.succeed(True)
for feed in data:
olddata = None
if self.is_cached(feed):
d.addCallback(self.get_feed_from_cache, feed)
d.addErrback(self.on_error, (feed, 'fetching from cache'))
else:
d.addCallback(self.get_page, feed)
d.addErrback(self.on_error, (feed, 'fetching'))
d.addCallback(self.parse_feed, feed)
d.addErrback(self.on_error, (feed, 'parsing'))
olddata = self.get_feed(feed)
d.addCallback(self.store_feed, feed)
d.addErrback(self.on_error, (feed, 'storing'))
d.addCallback(self.process_feed, feed, olddata)
d.addErrback(self.on_error, (feed, 'processing'))
del(olddata)
return d
def setUp(self):
self.hub = MokshaHub()
def setUp(self):
self.hub = MokshaHub()