示例#1
0
class TestHub:
    def setUp(self):
        self.hub = MokshaHub()

    def tearDown(self):
        self.hub.close()

    def test_hub_creation(self):
        assert_true(self.hub)
        eq_(self.hub.topics, {})
示例#2
0
class TestHub:

    def setUp(self):
        self.hub = MokshaHub()

    def tearDown(self):
        self.hub.close()

    def test_hub_creation(self):
        assert_true(self.hub)
        eq_(self.hub.topics, {})
示例#3
0
文件: streams.py 项目: decause/moksha
 def __init__(self):
     self.parsed = 1
     self.hub = MokshaHub()
     self.post_processors = []
     for entry in pkg_resources.iter_entry_points(
             'moksha.feeds.post_processor'):
         log.info('Registering feed post-processor: %s' % entry.name)
         self.post_processors.append(entry.load())
示例#4
0
 def __init__(self):
     self.parsed = 1
     self.hub = MokshaHub()
示例#5
0
class FeederProtocol(object):

    max_age = int(config.get('feed.max_age', 300))
    timeout = int(config.get('feed.timeout', 30))

    def __init__(self):
        self.parsed = 1
        self.hub = MokshaHub()

    def is_cached(self, site):
        already_got = feed_storage.get(site)
        if already_got:
            elapsed_time = time.time() - already_got[0]
            if elapsed_time < self.max_age:
                return True
            else:
                return False
        else:
            return False

    def on_error(self, traceback, extra_args):
        log.error(extra_args)
        log.exception(traceback)

    def get_feed_from_cache(self, data, key=None):
        """ Return feed data from the cache based on a given ``key`` """
        log.debug('Getting cached feed for %s' % key)
        return defer.succeed(feed_storage.get(key, key)[1])

    def parse_feed(self, feed, url):
        if not feed:
            log.warning('parse_feed got %r for %s' % (feed, url))
            return {}
        if not isinstance(feed, basestring):
            feed = str(feed)
        feed = feedparser.parse(StringIO(feed))
        assert feed
        if feed.get('bozo_exception'):
            bozo_exc = str(feed['bozo_exception'])
            log.warning("Feed %s getting bozo_exception %r" % (feed, bozo_exc))
            feed['bozo_exception'] = bozo_exc
        return feed

    def store_feed(self, feed, addr):
        feed_storage[addr] = (time.time(), feed)
        return feed

    def get_feed(self, addr):
        try:
            return feed_storage[addr][1]
        except KeyError:
            return None

    def process_feed(self, parsed_feed, addr, olddata):
        """ Process the parsed feed.

        If `olddata` is provided, this method will look for new feed entries,
        and send notifications to the `feed.$FEED_URL` MokshaHub Topic.

        :param parsed_feed: A parsed :mod:`feedcache` feed
        :param addr: The URL of the feed
        :param olddata: The cached feed data
        """
        if not parsed_feed:
            log.error("Cannot process %r feed for %s" % (parsed_feed, addr))
            return

        chan = parsed_feed.get('channel', None)
        if chan:
            log.debug(chan.get('title', ''))

        # Previous data provided; look for new entries.
        if olddata:
            oldtitles = [entry.get('title') for entry in olddata['entries']]
            new_entries = parsed_feed.get('entries', [{}])
            if not len(new_entries):
                log.warning('Feed contains empty entries: %s' % addr)
                return

            # If there are no new entries, move on...
            newtitle = new_entries[0].get('title', None)
            if newtitle == oldtitles[0]:
                return

            # Send notifications for each new entry
            for entry in new_entries[::-1]:
                entry_title = entry.get('title', '[No Title]')
                channel_link = entry.get('channel', {'link': addr})['link']
                if entry['title'] not in oldtitles:
                    log.info('New feed entry found: %s' % entry['title'])
                    self.hub.send_message(['feed.%s' % channel_link, 'feeds'],
                            {'title': entry_title, 'link': entry.get('link')})

    def get_page(self, data, args):
        return conditional_get_page(args, timeout=self.timeout)

    def start(self, data=None):
        d = defer.succeed(True)
        for feed in data:
            olddata = None
            if self.is_cached(feed):
                d.addCallback(self.get_feed_from_cache, feed)
                d.addErrback(self.on_error, (feed, 'fetching from cache'))
            else:
                d.addCallback(self.get_page, feed)
                d.addErrback(self.on_error, (feed, 'fetching'))
                d.addCallback(self.parse_feed, feed)
                d.addErrback(self.on_error, (feed, 'parsing'))
                olddata = self.get_feed(feed)
                d.addCallback(self.store_feed, feed)
                d.addErrback(self.on_error, (feed, 'storing'))
            d.addCallback(self.process_feed, feed, olddata)
            d.addErrback(self.on_error, (feed, 'processing'))
            del(olddata)
        return d
示例#6
0
 def setUp(self):
     self.hub = MokshaHub()
示例#7
0
 def setUp(self):
     self.hub = MokshaHub()