Example #1
0
    def __init__(self, host='localhost', port=6379, db=0, listen=False):
        """
        Start a new Thoonk instance for creating and managing feeds.

        Arguments:
            host   -- The Redis server name.
            port   -- Port for connecting to the Redis server.
            db     -- The Redis database to use.
            listen -- Flag indicating if this Thoonk instance should listen
                      for feed events and relevant event handlers. Defaults
                      to False.
        """
        self.host = host
        self.port = port
        self.db = db
        self.redis = redis.Redis(host=self.host, port=self.port, db=self.db)
        self.lredis = None

        self.feedtypes = {}
        self.feeds = set()
        self._feed_config = ConfigCache(self)
        self.handlers = {
                'create_notice': [],
                'delete_notice': [],
                'publish_notice': [],
                'retract_notice': [],
                'position_notice': [],
                'stalled_notice': [],
                'retried_notice': [],
                'finished_notice': [],
                'claimed_notice': [],
                'cancelled_notice': []}

        self.listen_ready = threading.Event()
        self.listening = listen

        self.feed_publish = 'feed.publish:%s'
        self.feed_retract = 'feed.retract:%s'
        self.feed_config = 'feed.config:%s'
        self.conf_feed = 'conffeed'
        self.new_feed = 'newfeed'
        self.del_feed = 'delfeed'

        self.register_feedtype(u'feed', feeds.Feed)
        self.register_feedtype(u'queue', feeds.Queue)
        self.register_feedtype(u'job', feeds.Job)
        self.register_feedtype(u'pyqueue', feeds.PythonQueue)
        self.register_feedtype(u'sorted_feed', feeds.SortedFeed)

        if listen:
            #start listener thread
            self.lthread = threading.Thread(target=self.listen)
            self.lthread.daemon = True
            self.lthread.start()
            self.listen_ready.wait()
Example #2
0
class Thoonk(object):

    """
    Thoonk provides a set of additional, high level datatypes with feed-like
    behaviour (feeds, queues, sorted feeds, job queues) to Redis. A default
    Thoonk instance will provide four feed types:
      feed       -- A simple list of entries sorted by publish date. May be
                    either bounded or bounded in size.
      queue      -- A feed that provides FIFO behaviour. Once an item is
                    pulled from the queue, it is removed.
      job        -- Similar to a queue, but an item is not removed from the
                    queue after it has been request until a job complete
                    notice is received.
      sorted feed -- Similar to a normal feed, except that the ordering of
                     items is not limited to publish date, and can be
                     manually adjusted.

    Thoonk.py also provides an additional pyqueue feed type which behaves
    identically to a queue, except that it pickles/unpickles Python
    datatypes automatically.

    The core Thoonk class provides infrastructure for creating and
    managing feeds.

    Attributes:
        db           -- The Redis database number.
        feeds        -- A set of known feed names.
        _feed_config   -- A cache of feed configurations.
        feedtypes    -- A dictionary mapping feed type names to their
                        implementation classes.
        handlers     -- A dictionary mapping event names to event handlers.
        host         -- The Redis server host.
        listen_ready -- A thread event indicating when the listening
                        Redis connection is ready.
        listening    -- A flag indicating if this Thoonk instance is for
                        listening to publish events.
        lredis       -- A Redis connection for listening to publish events.
        port         -- The Redis server port.
        redis        -- The Redis connection instance.

    Methods:
        close             -- Terminate the listening Redis connection.
        create_feed       -- Create a new feed using a given type and config.
        create_notice     -- Execute handlers for feed creation event.
        delete_feed       -- Remove an existing feed.
        delete_notice     -- Execute handlers for feed deletion event.
        feed_exists       -- Determine if a feed has already been created.
        get_feeds         -- Return the set of active feeds.
        listen            -- Start the listening Redis connection.
        publish_notice    -- Execute handlers for item publish event.
        register_feedtype -- Make a new feed type available for use.
        register_handler  -- Assign a function as an event handler.
        retract_notice    -- Execute handlers for item retraction event.
        set_config        -- Set the configuration for a given feed.
    """

    def __init__(self, host='localhost', port=6379, db=0, listen=False):
        """
        Start a new Thoonk instance for creating and managing feeds.

        Arguments:
            host   -- The Redis server name.
            port   -- Port for connecting to the Redis server.
            db     -- The Redis database to use.
            listen -- Flag indicating if this Thoonk instance should listen
                      for feed events and relevant event handlers. Defaults
                      to False.
        """
        self.host = host
        self.port = port
        self.db = db
        self.redis = redis.Redis(host=self.host, port=self.port, db=self.db)
        self.lredis = None

        self.feedtypes = {}
        self.feeds = set()
        self._feed_config = ConfigCache(self)
        self.handlers = {
                'create_notice': [],
                'delete_notice': [],
                'publish_notice': [],
                'retract_notice': [],
                'position_notice': [],
                'stalled_notice': [],
                'retried_notice': [],
                'finished_notice': [],
                'claimed_notice': [],
                'cancelled_notice': []}

        self.listen_ready = threading.Event()
        self.listening = listen

        self.feed_publish = 'feed.publish:%s'
        self.feed_retract = 'feed.retract:%s'
        self.feed_config = 'feed.config:%s'
        self.conf_feed = 'conffeed'
        self.new_feed = 'newfeed'
        self.del_feed = 'delfeed'

        self.register_feedtype(u'feed', feeds.Feed)
        self.register_feedtype(u'queue', feeds.Queue)
        self.register_feedtype(u'job', feeds.Job)
        self.register_feedtype(u'pyqueue', feeds.PythonQueue)
        self.register_feedtype(u'sorted_feed', feeds.SortedFeed)

        if listen:
            #start listener thread
            self.lthread = threading.Thread(target=self.listen)
            self.lthread.daemon = True
            self.lthread.start()
            self.listen_ready.wait()

    def _publish(self, schema, items):
        """
        A shortcut method to publish items separated by \x00.

        Arguments:
            schema -- The key to publish the items to.
            items  -- A tuple or list of items to publish.
        """
        self.redis.publish(schema, "\x00".join(items))

    def __getitem__(self, feed):
        """
        Return the configuration for a feed.

        Arguments:
            feed -- The name of the feed.

        Returns: Dict
        """
        return self._feed_config[feed]

    def __setitem__(self, feed, config):
        """
        Set the configuration for a feed.

        Arguments:
            feed   -- The name of the feed.
            config -- A dict of config values.
        """
        self.set_config(feed, config)

    def register_feedtype(self, feedtype, klass):
        """
        Make a new feed type availabe for use.

        New instances of the feed can be created by using:
            self.<feedtype>()

        For example: self.pyqueue() or self.job().

        Arguments:
            feedtype -- The name of the feed type.
            klass    -- The implementation class for the type.
        """
        self.feedtypes[feedtype] = klass

        def startclass(feed, config=None):
            """
            Instantiate a new feed on demand.

            Arguments:
                feed -- The name of the new feed.
                config -- A dictionary of configuration values.

            Returns: Feed of type <feedtype>.
            """
            if config is None:
                config = {}
            if self.feed_exists(feed):
                return self[feed]
            else:
                if not config.get('type', False):
                    config['type'] = feedtype
                return self.create_feed(feed, config)

        setattr(self, feedtype, startclass)

    def register_handler(self, name, handler):
        """
        Register a function to respond to feed events.

        Event types:
            - create_notice
            - delete_notice
            - publish_notice
            - retract_notice
            - position_notice

        Arguments:
            name    -- The name of the feed event.
            handler -- The function for handling the event.
        """
        if name not in self.handlers:
            self.handlers[name] = []
        self.handlers[name].append(handler)

    def remove_handler(self, name, handler):
        """
        Unregister a function that was registered via register_handler

        Arguments:
            name    -- The name of the feed event.
            handler -- The function for handling the event.
        """
        try:
            self.handlers[name].remove(handler)
        except (KeyError, ValueError):
            pass
        

    def create_feed(self, feed, config):
        """
        Create a new feed with a given configuration.

        The configuration is a dict, and should include a 'type'
        entry with the class of the feed type implementation.

        Arguments:
            feed   -- The name of the new feed.
            config -- A dictionary of configuration values.
        """
        if config is None:
            config = {}
        if not self.redis.sadd("feeds", feed):
            raise FeedExists
        self.feeds.add(feed)
        self.set_config(feed, config)
        self._publish(self.new_feed, (feed, self._feed_config.instance))
        return self[feed]

    def delete_feed(self, feed):
        """
        Delete a given feed.

        Arguments:
            feed -- The name of the feed.
        """
        feed_instance = self._feed_config[feed]
        deleted = False
        while not deleted:
            self.redis.watch('feeds')
            if not self.feed_exists(feed):
                return FeedDoesNotExist
            pipe = self.redis.pipeline()
            pipe.srem("feeds", feed)
            for key in feed_instance.get_schemas():
                pipe.delete(key)
                self._publish(self.del_feed, (feed, self._feed_config.instance))
            try:
                pipe.execute()
                deleted = True
            except redis.exceptions.WatchError:
                deleted = False

    def set_config(self, feed, config):
        """
        Set the configuration for a given feed.

        Arguments:
            feed   -- The name of the feed.
            config -- A dictionary of configuration values.
        """
        if not self.feed_exists(feed):
            raise FeedDoesNotExist
        if type(config) == dict:
            if u'type' not in config:
                config[u'type'] = u'feed'
            jconfig = json.dumps(config)
            dconfig = config
        else:
            dconfig = json.loads(config)
            if u'type' not in dconfig:
                dconfig[u'type'] = u'feed'
            jconfig = json.dumps(dconfig)
        self.redis.set(self.feed_config % feed, jconfig)
        self._publish(self.conf_feed, (feed, self._feed_config.instance))

    def get_config(self, feed):
        if not self.feed_exists(feed):
            raise FeedDoesNotExist
        config = self.redis.get(self.feed_config % feed)
        return json.loads(config)

    def get_feeds(self):
        """
        Return the set of known feeds.

        Returns: set
        """
        self.feeds.update(self.redis.smembers('feeds'))
        return self.feeds

    def feed_exists(self, feed):
        """
        Check if a given feed exists.

        Arguments:
            feed -- The name of the feed.
        """
        if not self.listening:
            if not feed in self.feeds:
                if self.redis.sismember('feeds', feed):
                    self.feeds.add(feed)
                    return True
                return False
            else:
                return True
        return feed in self.feeds

    def close(self):
        """Terminate the listening Redis connection."""
        self.redis.connection.disconnect()
        if self.listening:
            self.lredis.connection.disconnect()

    def listen(self):
        """
        Listen for feed creation and manipulation events and execute
        relevant event handlers. Specifically, listen for:
            - Feed creations
            - Feed deletions
            - Configuration changes
            - Item publications.
            - Item retractions.
        """
        # listener redis object
        self.lredis = redis.Redis(host=self.host, port=self.port, db=self.db)

        # subscribe to feed activities channel
        self.lredis.subscribe((self.new_feed, self.del_feed, self.conf_feed))

        # get set of feeds
        feeds = self.get_feeds()
        # subscribe to exist feeds retract and publish
        for feed in self.feeds:
            self.lredis.subscribe(self[feed].get_channels())

        self.listen_ready.set()
        for event in self.lredis.listen():
            if event['type'] == 'message':
                if event['channel'].startswith('feed.publish'):
                    #feed publish event
                    id, item = event['data'].split('\x00', 1)
                    self.publish_notice(event['channel'].split(':', 1)[-1],
                                        item, id)
                elif event['channel'].startswith('feed.retract'):
                    self.retract_notice(event['channel'].split(':', 1)[-1],
                                        event['data'])
                elif event['channel'].startswith('feed.position'):
                    self.position_notice(event['channel'].split(':', 1)[-1],
                                         event['data'])
                elif event['channel'].startswith('feed.claimed'):
                    self.claimed_notice(event['channel'].split(':', 1)[-1],
                                        event['data'])
                elif event['channel'].startswith('feed.finished'):
                    id, data = event['data'].split(':', 1)[-1].split("\x00", 1)
                    self.finished_notice(event['channel'].split(':', 1)[-1], id,
                                        data)
                elif event['channel'].startswith('feed.cancelled'):
                    self.cancelled_notice(event['channel'].split(':', 1)[-1],
                                        event['data'])
                elif event['channel'].startswith('feed.stalled'):
                    self.stalled_notice(event['channel'].split(':', 1)[-1],
                                        event['data'])
                elif event['channel'].startswith('feed.retried'):
                    self.retried_notice(event['channel'].split(':', 1)[-1],
                                        event['data'])
                elif event['channel'] == self.new_feed:
                    #feed created event
                    name, instance = event['data'].split('\x00')
                    self.feeds.add(name)
                    config = self.get_config(name)
                    if config["type"] == "job":
                        self.lredis.subscribe(("feed.publishes:%s" % name,
                                               "feed.cancelled:%s" % name,
                                               "feed.claimed:%s" % name,
                                               "feed.finished:%s" % name,
                                               "feed.stalled:%s" % name,))
                    else:
                        self.lredis.subscribe((self.feed_publish % name,
                                               self.feed_retract % name))
                    self.create_notice(name)
                elif event['channel'] == self.del_feed:
                    #feed destroyed event
                    name, instance = event['data'].split('\x00')
                    try:
                        self.feeds.remove(name)
                    except KeyError:
                        #already removed -- probably locally
                        pass
                    self._feed_config.invalidate(name, instance, delete=True)
                    self.delete_notice(name)
                elif event['channel'] == self.conf_feed:
                    feed, instance = event['data'].split('\x00', 1)
                    self._feed_config.invalidate(feed, instance)

    def create_notice(self, feed):
        """
        Generate a notice that a new feed has been created and
        execute any relevant event handlers.

        Arguments:
            feed -- The name of the created feed.
        """
        for handler in self.handlers['create_notice']:
            handler(feed)

    def delete_notice(self, feed):
        """
        Generate a notice that a feed has been deleted, and
        execute any relevant event handlers.

        Arguments:
            feed -- The name of the deleted feed.
        """
        for handler in self.handlers['delete_notice']:
            handler(feed)

    def publish_notice(self, feed, item, id):
        """
        Generate a notice that an item has been published to a feed, and
        execute any relevant event handlers.

        Arguments:
            feed -- The name of the feed.
            item -- The content of the published item.
            id   -- The ID of the published item.
        """
        self[feed].event_publish(id, item)
        for handler in self.handlers['publish_notice']:
            handler(feed, item, id)

    def retract_notice(self, feed, id):
        """
        Generate a notice that an item has been retracted from a feed, and
        execute any relevant event handlers.

        Arguments:
            feed -- The name of the feed.
            id   -- The ID of the retracted item.
        """
        self[feed].event_retract(id)
        for handler in self.handlers['retract_notice']:
            handler(feed, id)

    def position_notice(self, feed, id, rel_id):
        """
        Generate a notice that an item has been moved, and
        execute any relevant event handlers.

        Arguments:
            feed   -- The name of the feed.
            id     -- The ID of the moved item.
            rel_id -- Where the item was moved, in relation to
                      existing items.
        """
        for handler in self.handlers['position_notice']:
            handler(feed, id, rel_id)
    
    def stalled_notice(self, feed, id):
        """
        Generate a notice that a job has been stalled, and
        execute any relevant event handlers.

        Arguments:
            feed   -- The name of the feed.
            id     -- The ID of the stalled item.
        """
        for handler in self.handlers['stalled_notice']:
            handler(feed, id)

    def retried_notice(self, feed, id):
        """
        Generate a notice that a job has been retried, and
        execute any relevant event handlers.

        Arguments:
            feed   -- The name of the feed.
            id     -- The ID of the retried item.
        """
        for handler in self.handlers['retried_notice']:
            handler(feed, id)

    def cancelled_notice(self, feed, id):
        """
        Generate a notice that a job has been cancelled, and
        execute any relevant event handlers.

        Arguments:
            feed   -- The name of the feed.
            id     -- The ID of the stalled item.
        """
        for handler in self.handlers['cancelled_notice']:
            handler(feed, id)

    def finished_notice(self, feed, id, result):
        """
        Generate a notice that a job has finished, and
        execute any relevant event handlers.

        Arguments:
            feed   -- The name of the feed.
            id     -- The ID of the stalled item.
        """
        for handler in self.handlers['finished_notice']:
            handler(feed, id, result)

    def claimed_notice(self, feed, id):
        """
        Generate a notice that a job has been claimed, and
        execute any relevant event handlers.

        Arguments:
            feed   -- The name of the feed.
            id     -- The ID of the stalled item.
        """
        for handler in self.handlers['claimed_notice']:
            handler(feed, id)