Esempio n. 1
0
    def setup_middleware(self):
        """
        Middleware setup happens here.

        Subclasses should not override this unless they need to do nonstandard
        middleware setup.
        """
        middlewares = yield setup_middlewares_from_config(self, self.config)
        self._middlewares = MiddlewareStack(middlewares)
Esempio n. 2
0
 def __init__(self,
              worker,
              connector_name,
              prefetch_count=None,
              middlewares=None):
     self.name = connector_name
     self.worker = worker
     self._consumers = {}
     self._publishers = {}
     self._endpoint_handlers = {}
     self._default_handlers = {}
     self._prefetch_count = prefetch_count
     self._middlewares = MiddlewareStack(
         middlewares if middlewares is not None else [])
Esempio n. 3
0
    def setup_middleware(self):
        """
        Middleware setup happens here.

        Subclasses should not override this unless they need to do nonstandard
        middleware setup.
        """
        middlewares = yield setup_middlewares_from_config(self, self.config)
        self._middlewares = MiddlewareStack(middlewares)
Esempio n. 4
0
 def __init__(self, worker, connector_name, prefetch_count=None,
              middlewares=None):
     self.name = connector_name
     self.worker = worker
     self._consumers = {}
     self._publishers = {}
     self._endpoint_handlers = {}
     self._default_handlers = {}
     self._prefetch_count = prefetch_count
     self._middlewares = MiddlewareStack(middlewares
                                         if middlewares is not None else [])
Esempio n. 5
0
 def __init__(self, config):
     self.transport_publisher = {}
     self.transport_names = config.get('transport_names', [])
     for transport in self.transport_names:
         self.transport_publisher[transport] = self.DummyPublisher()
     self.exposed_publisher = {}
     self.exposed_event_publisher = {}
     self.exposed_names = config.get('exposed_names', [])
     for exposed in self.exposed_names:
         self.exposed_publisher[exposed] = self.DummyPublisher()
         self.exposed_event_publisher[exposed] = self.DummyPublisher()
     self._middlewares = MiddlewareStack([])
Esempio n. 6
0
class BaseConnector(object):
    """Base class for 'connector' objects.

    A connector encapsulates the 'inbound', 'outbound' and 'event' publishers
    and consumers required by vumi workers and avoids having to operate on them
    individually all over the place.
    """
    def __init__(self, worker, connector_name, prefetch_count=None,
                 middlewares=None):
        self.name = connector_name
        self.worker = worker
        self._consumers = {}
        self._publishers = {}
        self._endpoint_handlers = {}
        self._default_handlers = {}
        self._prefetch_count = prefetch_count
        self._middlewares = MiddlewareStack(middlewares
                                            if middlewares is not None else [])

    def _rkey(self, mtype):
        return '%s.%s' % (self.name, mtype)

    def setup(self):
        raise NotImplementedError()

    def teardown(self):
        d = gatherResults([c.stop() for c in self._consumers.values()])
        d.addCallback(lambda r: self._middlewares.teardown())
        return d

    @property
    def paused(self):
        return all(consumer.paused
                   for consumer in self._consumers.itervalues())

    def pause(self):
        return gatherResults([
            consumer.pause() for consumer in self._consumers.itervalues()])

    def unpause(self):
        # This doesn't return a deferred.
        for consumer in self._consumers.values():
            consumer.unpause()

    @inlineCallbacks
    def _setup_publisher(self, mtype):
        publisher = yield self.worker.publish_to(self._rkey(mtype))
        self._publishers[mtype] = publisher
        returnValue(publisher)

    @inlineCallbacks
    def _setup_consumer(self, mtype, msg_class, default_handler):
        def handler(msg):
            return self._consume_message(mtype, msg)

        consumer = yield self.worker.consume(
            self._rkey(mtype), handler, message_class=msg_class, paused=True,
            prefetch_count=self._prefetch_count)
        self._consumers[mtype] = consumer
        self._set_default_endpoint_handler(mtype, default_handler)
        returnValue(consumer)

    def _set_endpoint_handler(self, mtype, handler, endpoint_name):
        if endpoint_name is None:
            endpoint_name = TransportMessage.DEFAULT_ENDPOINT_NAME
        handlers = self._endpoint_handlers.setdefault(mtype, {})
        handlers[endpoint_name] = handler

    def _set_default_endpoint_handler(self, mtype, handler):
        self._endpoint_handlers.setdefault(mtype, {})
        self._default_handlers[mtype] = handler

    def _consume_message(self, mtype, msg):
        endpoint_name = msg.get_routing_endpoint()
        handler = self._endpoint_handlers[mtype].get(endpoint_name)
        if handler is None:
            handler = self._default_handlers.get(mtype)
        d = self._middlewares.apply_consume(mtype, msg, self.name)
        d.addCallback(handler)
        return d.addErrback(self._ignore_message, msg)

    def _publish_message(self, mtype, msg, endpoint_name):
        if endpoint_name is not None:
            msg.set_routing_endpoint(endpoint_name)
        d = self._middlewares.apply_publish(mtype, msg, self.name)
        return d.addCallback(self._publishers[mtype].publish_message)

    def _ignore_message(self, failure, msg):
        failure.trap(IgnoreMessage)
        log.debug("Ignoring msg due to %r: %r" % (failure.value, msg))
Esempio n. 7
0
class Transport(Worker):
    """
    Base class for transport workers.

    The following attributes are available for subclasses to control behaviour:

    * :attr:`start_message_consumer` -- Set to ``False`` if the message
      consumer should not be started. The subclass is responsible for starting
      it in this case.
    """

    SUPPRESS_FAILURE_EXCEPTIONS = True

    transport_name = None
    start_message_consumer = True

    @inlineCallbacks
    def startWorker(self):
        """
        Set up basic transport worker stuff.

        You shouldn't have to override this in subclasses.
        """
        self._consumers = []

        self._validate_config()
        if 'TRANSPORT_NAME' in self.config:
            log.msg("NOTE: 'TRANSPORT_NAME' in config is deprecated. "
                    "Use 'transport_name' instead.")
            self.config.setdefault('transport_name',
                                   self.config['TRANSPORT_NAME'])
        self.transport_name = self.config['transport_name']
        self.concurrent_sends = self.config.get('concurrent_sends')

        yield self._setup_failure_publisher()
        yield self._setup_message_publisher()
        yield self._setup_event_publisher()

        yield self.setup_middleware()

        yield self.setup_transport()

        self.message_consumer = None
        if self.start_message_consumer:
            yield self._setup_message_consumer()

    @inlineCallbacks
    def stopWorker(self):
        while self._consumers:
            consumer = self._consumers.pop()
            yield consumer.stop()
        yield self.teardown_transport()

    def get_rkey(self, mtype):
        return '%s.%s' % (self.transport_name, mtype)

    def publish_rkey(self, name):
        return self.publish_to(self.get_rkey(name))

    def _validate_config(self):
        if 'transport_name' not in self.config:
            raise ConfigError("Missing 'transport_name' field in config.")
        return self.validate_config()

    def validate_config(self):
        """
        Transport-specific config validation happens in here.

        Subclasses may override this method to perform extra config validation.
        """
        pass

    def setup_transport(self):
        """
        All transport_specific setup should happen in here.

        Subclasses should override this method to perform extra setup.
        """
        pass

    def teardown_transport(self):
        """
        Clean-up of setup done in setup_transport should happen here.
        """
        pass

    @inlineCallbacks
    def setup_middleware(self):
        """
        Middleware setup happens here.

        Subclasses should not override this unless they need to do nonstandard
        middleware setup.
        """
        middlewares = yield setup_middlewares_from_config(self, self.config)
        self._middlewares = MiddlewareStack(middlewares)

    @inlineCallbacks
    def _setup_message_publisher(self):
        self.message_publisher = yield self.publish_rkey('inbound')

    @inlineCallbacks
    def _setup_message_consumer(self):
        if self.message_consumer is not None:
            log.msg("Consumer already exists, not restarting.")
            return

        self.message_consumer = yield self.consume(
            self.get_rkey('outbound'),
            self._process_message,
            message_class=TransportUserMessage)
        self._consumers.append(self.message_consumer)

        # Apply concurrency throttling if we need to.
        if self.concurrent_sends is not None:
            yield self.message_consumer.channel.basic_qos(
                0, int(self.concurrent_sends), False)

    def _teardown_message_consumer(self):
        if self.message_consumer is None:
            log.msg("Consumer does not exist, not stopping.")
            return
        if self.message_consumer in self._consumers:
            self._consumers.remove(self.message_consumer)
        consumer, self.message_consumer = self.message_consumer, None
        return consumer.stop()

    @inlineCallbacks
    def _setup_event_publisher(self):
        self.event_publisher = yield self.publish_rkey('event')

    @inlineCallbacks
    def _setup_failure_publisher(self):
        self.failure_publisher = yield self.publish_rkey('failures')

    def send_failure(self, message, exception, traceback):
        """Send a failure report."""
        try:
            failure_code = getattr(exception, "failure_code",
                                   FailureMessage.FC_UNSPECIFIED)
            failure_msg = FailureMessage(message=message.payload,
                                         failure_code=failure_code,
                                         reason=traceback)
            d = self._middlewares.apply_publish("failure", failure_msg,
                                                self.transport_name)
            d.addCallback(self.failure_publisher.publish_message)
            d.addCallback(lambda _f: self.failure_published())
        except:
            log.err("Error publishing failure: %s, %s, %s" %
                    (message, exception, traceback))
            raise
        return d

    def failure_published(self):
        pass

    def publish_message(self, **kw):
        """
        Publish a :class:`TransportUserMessage` message.

        Some default parameters are handled, so subclasses don't have
        to provide a lot of boilerplate.
        """
        kw.setdefault('transport_name', self.transport_name)
        kw.setdefault('transport_metadata', {})
        msg = TransportUserMessage(**kw)
        d = self._middlewares.apply_publish("inbound", msg,
                                            self.transport_name)
        d.addCallback(self.message_publisher.publish_message)
        return d

    def publish_event(self, **kw):
        """
        Publish a :class:`TransportEvent` message.

        Some default parameters are handled, so subclasses don't have
        to provide a lot of boilerplate.
        """
        kw.setdefault('transport_name', self.transport_name)
        kw.setdefault('transport_metadata', {})
        event = TransportEvent(**kw)
        d = self._middlewares.apply_publish("event", event,
                                            self.transport_name)
        d.addCallback(self.event_publisher.publish_message)
        return d

    def publish_ack(self, user_message_id, sent_message_id, **kw):
        """
        Helper method for publishing an ``ack`` event.
        """
        return self.publish_event(user_message_id=user_message_id,
                                  sent_message_id=sent_message_id,
                                  event_type='ack',
                                  **kw)

    def publish_delivery_report(self, user_message_id, delivery_status, **kw):
        """
        Helper method for publishing a ``delivery_report`` event.
        """
        return self.publish_event(user_message_id=user_message_id,
                                  delivery_status=delivery_status,
                                  event_type='delivery_report',
                                  **kw)

    def _process_message(self, message):
        def _send_failure(f):
            self.send_failure(message, f.value, f.getTraceback())
            log.err(f)
            if self.SUPPRESS_FAILURE_EXCEPTIONS:
                return None
            return f

        d = self._middlewares.apply_consume("outbound", message,
                                            self.transport_name)
        d.addCallback(self.handle_outbound_message)
        d.addErrback(_send_failure)
        return d

    def handle_outbound_message(self, message):
        """
        This must be overridden to read outbound messages and do the right
        thing with them.
        """
        raise NotImplementedError()

    @staticmethod
    def generate_message_id():
        """
        Generate a message id.
        """

        return TransportUserMessage.generate_id()
Esempio n. 8
0
class ApplicationWorker(Worker):
    """Base class for an application worker.

    Handles :class:`vumi.message.TransportUserMessage` and
    :class:`vumi.message.TransportEvent` messages.

    Application workers may send outgoing messages using
    :meth:`reply_to` (for replies to incoming messages) or
    :meth:`send_to` (for messages that are not replies).

    Messages sent via :meth:`send_to` pass optional additional data
    from configuration to the TransportUserMessage constructor, based
    on the tag parameter passed to send_to. This usually contains
    information useful for routing the message.

    An example :meth:`send_to` configuration might look like::

      - send_to:
        - default:
          transport_name: sms_transport

    Currently 'transport_name' **must** be defined for each send_to
    section since all existing dispatchers rely on this for routing
    outbound messages.

    The available tags are defined by the :attr:`SEND_TO_TAGS` class
    attribute. Sub-classes must override this attribute with a set of
    tag names if they wish to use :meth:`send_to`. If applications
    have only a single tag, it is suggested to name that tag `default`
    (this makes calling `send_to` easier since the value of the tag
    parameter may be omitted).

    By default :attr:`SEND_TO_TAGS` is empty and all calls to
    :meth:`send_to` will fail (this is to make it easy to identify
    which tags an application requires `send_to` configuration for).
    """

    transport_name = None
    start_message_consumer = True

    SEND_TO_TAGS = frozenset([])

    @inlineCallbacks
    def startWorker(self):
        log.msg('Starting a %s worker with config: %s'
                % (self.__class__.__name__, self.config))
        self._consumers = []
        self._validate_config()
        self.transport_name = self.config['transport_name']
        self.send_to_options = self.config.get('send_to', {})

        self._event_handlers = {
            'ack': self.consume_ack,
            'delivery_report': self.consume_delivery_report,
            }
        self._session_handlers = {
            SESSION_NEW: self.new_session,
            SESSION_CLOSE: self.close_session,
            }

        yield self._setup_transport_publisher()

        yield self.setup_middleware()

        yield self.setup_application()

        if self.start_message_consumer:
            yield self._setup_transport_consumer()
            yield self._setup_event_consumer()

    @inlineCallbacks
    def stopWorker(self):
        while self._consumers:
            consumer = self._consumers.pop()
            yield consumer.stop()
        yield self.teardown_application()

    def _validate_config(self):
        if 'transport_name' not in self.config:
            raise ConfigError("Missing 'transport_name' field in config.")
        send_to_options = self.config.get('send_to', {})
        for tag in self.SEND_TO_TAGS:
            if tag not in send_to_options:
                raise ConfigError("No configuration for send_to tag %r but"
                                  " at least a transport_name is required."
                                  % (tag,))
            if 'transport_name' not in send_to_options[tag]:
                raise ConfigError("The configuration for send_to tag %r must"
                                  " contain a transport_name." % (tag,))

        return self.validate_config()

    def validate_config(self):
        """
        Application-specific config validation happens in here.

        Subclasses may override this method to perform extra config validation.
        """
        pass

    def setup_application(self):
        """
        All application specific setup should happen in here.

        Subclasses should override this method to perform extra setup.
        """
        pass

    def teardown_application(self):
        """
        Clean-up of setup done in setup_application should happen here.
        """
        pass

    @inlineCallbacks
    def setup_middleware(self):
        """
        Middleware setup happens here.

        Subclasses should not override this unless they need to do nonstandard
        middleware setup.
        """
        middlewares = yield setup_middlewares_from_config(self, self.config)
        self._middlewares = MiddlewareStack(middlewares)

    def _dispatch_event_raw(self, event):
        event_type = event.get('event_type')
        handler = self._event_handlers.get(event_type,
                                           self.consume_unknown_event)
        return handler(event)

    def dispatch_event(self, event):
        """Dispatch to event_type specific handlers."""
        d = self._middlewares.apply_consume("event", event,
                                            self.transport_name)
        d.addCallback(self._dispatch_event_raw)
        return d

    def consume_unknown_event(self, event):
        log.msg("Unknown event type in message %r" % (event,))

    def consume_ack(self, event):
        """Handle an ack message."""
        pass

    def consume_delivery_report(self, event):
        """Handle a delivery report."""
        pass

    def _dispatch_user_message_raw(self, message):
        session_event = message.get('session_event')
        handler = self._session_handlers.get(session_event,
                                             self.consume_user_message)
        return handler(message)

    def dispatch_user_message(self, message):
        """Dispatch user messages to handler."""
        d = self._middlewares.apply_consume("inbound", message,
                                            self.transport_name)
        d.addCallback(self._dispatch_user_message_raw)
        return d

    def consume_user_message(self, message):
        """Respond to user message."""
        pass

    def new_session(self, message):
        """Respond to a new session.

        Defaults to calling consume_user_message.
        """
        return self.consume_user_message(message)

    def close_session(self, message):
        """Close a session.

        The .reply_to() method should not be called when the session is closed.
        """
        pass

    def _publish_message(self, message):
        d = self._middlewares.apply_publish("outbound", message,
                                            self.transport_name)
        d.addCallback(self.transport_publisher.publish_message)
        return d

    def reply_to(self, original_message, content, continue_session=True,
                 **kws):
        reply = original_message.reply(content, continue_session, **kws)
        return self._publish_message(reply)

    def reply_to_group(self, original_message, content, continue_session=True,
                       **kws):
        reply = original_message.reply_group(content, continue_session, **kws)
        return self._publish_message(reply)

    def send_to(self, to_addr, content, tag='default', **kw):
        if tag not in self.SEND_TO_TAGS:
            raise ValueError("Tag %r not defined in SEND_TO_TAGS" % (tag,))
        options = copy.deepcopy(self.send_to_options[tag])
        options.update(kw)
        msg = TransportUserMessage.send(to_addr, content, **options)
        return self._publish_message(msg)

    @inlineCallbacks
    def _setup_transport_publisher(self):
        self.transport_publisher = yield self.publish_to(
            '%(transport_name)s.outbound' % self.config)

    @inlineCallbacks
    def _setup_transport_consumer(self):
        self.transport_consumer = yield self.consume(
            '%(transport_name)s.inbound' % self.config,
            self.dispatch_user_message,
            message_class=TransportUserMessage)
        self._consumers.append(self.transport_consumer)

    @inlineCallbacks
    def _setup_event_consumer(self):
        self.transport_event_consumer = yield self.consume(
            '%(transport_name)s.event' % self.config,
            self.dispatch_event,
            message_class=TransportEvent)
        self._consumers.append(self.transport_event_consumer)
Esempio n. 9
0
class ApplicationWorker(Worker):
    """Base class for an application worker.

    Handles :class:`vumi.message.TransportUserMessage` and
    :class:`vumi.message.TransportEvent` messages.

    Application workers may send outgoing messages using
    :meth:`reply_to` (for replies to incoming messages) or
    :meth:`send_to` (for messages that are not replies).

    Messages sent via :meth:`send_to` pass optional additional data
    from configuration to the TransportUserMessage constructor, based
    on the tag parameter passed to send_to. This usually contains
    information useful for routing the message.

    An example :meth:`send_to` configuration might look like::

      - send_to:
        - default:
          transport_name: sms_transport

    Currently 'transport_name' **must** be defined for each send_to
    section since all existing dispatchers rely on this for routing
    outbound messages.

    The available tags are defined by the :attr:`SEND_TO_TAGS` class
    attribute. Sub-classes must override this attribute with a set of
    tag names if they wish to use :meth:`send_to`. If applications
    have only a single tag, it is suggested to name that tag `default`
    (this makes calling `send_to` easier since the value of the tag
    parameter may be omitted).

    By default :attr:`SEND_TO_TAGS` is empty and all calls to
    :meth:`send_to` will fail (this is to make it easy to identify
    which tags an application requires `send_to` configuration for).
    """

    transport_name = None
    start_message_consumer = True

    SEND_TO_TAGS = frozenset([])

    @inlineCallbacks
    def startWorker(self):
        log.msg('Starting a %s worker with config: %s' %
                (self.__class__.__name__, self.config))
        self._consumers = []
        self._validate_config()
        self.transport_name = self.config['transport_name']
        self.send_to_options = self.config.get('send_to', {})

        self._event_handlers = {
            'ack': self.consume_ack,
            'delivery_report': self.consume_delivery_report,
        }
        self._session_handlers = {
            SESSION_NEW: self.new_session,
            SESSION_CLOSE: self.close_session,
        }

        yield self._setup_transport_publisher()

        yield self.setup_middleware()

        yield self.setup_application()

        if self.start_message_consumer:
            yield self._setup_transport_consumer()
            yield self._setup_event_consumer()

    @inlineCallbacks
    def stopWorker(self):
        while self._consumers:
            consumer = self._consumers.pop()
            yield consumer.stop()
        yield self.teardown_application()

    def _validate_config(self):
        if 'transport_name' not in self.config:
            raise ConfigError("Missing 'transport_name' field in config.")
        send_to_options = self.config.get('send_to', {})
        for tag in self.SEND_TO_TAGS:
            if tag not in send_to_options:
                raise ConfigError("No configuration for send_to tag %r but"
                                  " at least a transport_name is required." %
                                  (tag, ))
            if 'transport_name' not in send_to_options[tag]:
                raise ConfigError("The configuration for send_to tag %r must"
                                  " contain a transport_name." % (tag, ))

        return self.validate_config()

    def validate_config(self):
        """
        Application-specific config validation happens in here.

        Subclasses may override this method to perform extra config validation.
        """
        pass

    def setup_application(self):
        """
        All application specific setup should happen in here.

        Subclasses should override this method to perform extra setup.
        """
        pass

    def teardown_application(self):
        """
        Clean-up of setup done in setup_application should happen here.
        """
        pass

    @inlineCallbacks
    def setup_middleware(self):
        """
        Middleware setup happens here.

        Subclasses should not override this unless they need to do nonstandard
        middleware setup.
        """
        middlewares = yield setup_middlewares_from_config(self, self.config)
        self._middlewares = MiddlewareStack(middlewares)

    def _dispatch_event_raw(self, event):
        event_type = event.get('event_type')
        handler = self._event_handlers.get(event_type,
                                           self.consume_unknown_event)
        return handler(event)

    def dispatch_event(self, event):
        """Dispatch to event_type specific handlers."""
        d = self._middlewares.apply_consume("event", event,
                                            self.transport_name)
        d.addCallback(self._dispatch_event_raw)
        return d

    def consume_unknown_event(self, event):
        log.msg("Unknown event type in message %r" % (event, ))

    def consume_ack(self, event):
        """Handle an ack message."""
        pass

    def consume_delivery_report(self, event):
        """Handle a delivery report."""
        pass

    def _dispatch_user_message_raw(self, message):
        session_event = message.get('session_event')
        handler = self._session_handlers.get(session_event,
                                             self.consume_user_message)
        return handler(message)

    def dispatch_user_message(self, message):
        """Dispatch user messages to handler."""
        d = self._middlewares.apply_consume("inbound", message,
                                            self.transport_name)
        d.addCallback(self._dispatch_user_message_raw)
        return d

    def consume_user_message(self, message):
        """Respond to user message."""
        pass

    def new_session(self, message):
        """Respond to a new session.

        Defaults to calling consume_user_message.
        """
        return self.consume_user_message(message)

    def close_session(self, message):
        """Close a session.

        The .reply_to() method should not be called when the session is closed.
        """
        pass

    def _publish_message(self, message):
        d = self._middlewares.apply_publish("outbound", message,
                                            self.transport_name)
        d.addCallback(self.transport_publisher.publish_message)
        return d

    def reply_to(self,
                 original_message,
                 content,
                 continue_session=True,
                 **kws):
        reply = original_message.reply(content, continue_session, **kws)
        return self._publish_message(reply)

    def reply_to_group(self,
                       original_message,
                       content,
                       continue_session=True,
                       **kws):
        reply = original_message.reply_group(content, continue_session, **kws)
        return self._publish_message(reply)

    def send_to(self, to_addr, content, tag='default', **kw):
        if tag not in self.SEND_TO_TAGS:
            raise ValueError("Tag %r not defined in SEND_TO_TAGS" % (tag, ))
        options = copy.deepcopy(self.send_to_options[tag])
        options.update(kw)
        msg = TransportUserMessage.send(to_addr, content, **options)
        return self._publish_message(msg)

    @inlineCallbacks
    def _setup_transport_publisher(self):
        self.transport_publisher = yield self.publish_to(
            '%(transport_name)s.outbound' % self.config)

    @inlineCallbacks
    def _setup_transport_consumer(self):
        self.transport_consumer = yield self.consume(
            '%(transport_name)s.inbound' % self.config,
            self.dispatch_user_message,
            message_class=TransportUserMessage)
        self._consumers.append(self.transport_consumer)

    @inlineCallbacks
    def _setup_event_consumer(self):
        self.transport_event_consumer = yield self.consume(
            '%(transport_name)s.event' % self.config,
            self.dispatch_event,
            message_class=TransportEvent)
        self._consumers.append(self.transport_event_consumer)
Esempio n. 10
0
 def setup_middleware(self):
     middlewares = yield setup_middlewares_from_config(self, self.config)
     self._middlewares = MiddlewareStack(middlewares)
Esempio n. 11
0
class BaseDispatchWorker(Worker):
    """Base class for a dispatch worker.

    """
    @inlineCallbacks
    def startWorker(self):
        log.msg('Starting a %s dispatcher with config: %s' %
                (self.__class__.__name__, self.config))

        self.amqp_prefetch_count = self.config.get('amqp_prefetch_count', 20)
        yield self.setup_endpoints()
        yield self.setup_middleware()
        yield self.setup_router()
        yield self.setup_transport_publishers()
        yield self.setup_exposed_publishers()
        yield self.setup_transport_consumers()
        yield self.setup_exposed_consumers()

        consumers = (self.exposed_consumer.values() +
                     self.transport_consumer.values() +
                     self.transport_event_consumer.values())
        for consumer in consumers:
            consumer.unpause()

    @inlineCallbacks
    def stopWorker(self):
        yield self.teardown_router()
        yield self.teardown_middleware()

    def setup_endpoints(self):
        self.transport_names = self.config.get('transport_names', [])
        self.exposed_names = self.config.get('exposed_names', [])

    @inlineCallbacks
    def setup_middleware(self):
        middlewares = yield setup_middlewares_from_config(self, self.config)
        self._middlewares = MiddlewareStack(middlewares)

    def teardown_middleware(self):
        return self._middlewares.teardown()

    def setup_router(self):
        router_cls = load_class_by_string(self.config['router_class'])
        self._router = router_cls(self, self.config)
        return maybeDeferred(self._router.setup_routing)

    def teardown_router(self):
        return maybeDeferred(self._router.teardown_routing)

    @inlineCallbacks
    def setup_transport_publishers(self):
        self.transport_publisher = {}
        for transport_name in self.transport_names:
            self.transport_publisher[transport_name] = yield self.publish_to(
                '%s.outbound' % (transport_name, ))

    @inlineCallbacks
    def setup_transport_consumers(self):
        self.transport_consumer = {}
        self.transport_event_consumer = {}
        for transport_name in self.transport_names:
            self.transport_consumer[transport_name] = yield self.consume(
                '%s.inbound' % (transport_name, ),
                functools.partial(self.dispatch_inbound_message,
                                  transport_name),
                message_class=TransportUserMessage,
                paused=True,
                prefetch_count=self.amqp_prefetch_count)
        for transport_name in self.transport_names:
            self.transport_event_consumer[transport_name] = yield self.consume(
                '%s.event' % (transport_name, ),
                functools.partial(self.dispatch_inbound_event, transport_name),
                message_class=TransportEvent,
                paused=True,
                prefetch_count=self.amqp_prefetch_count)

    @inlineCallbacks
    def setup_exposed_publishers(self):
        self.exposed_publisher = {}
        self.exposed_event_publisher = {}
        for exposed_name in self.exposed_names:
            self.exposed_publisher[exposed_name] = yield self.publish_to(
                '%s.inbound' % (exposed_name, ))
        for exposed_name in self.exposed_names:
            self.exposed_event_publisher[exposed_name] = yield self.publish_to(
                '%s.event' % (exposed_name, ))

    @inlineCallbacks
    def setup_exposed_consumers(self):
        self.exposed_consumer = {}
        for exposed_name in self.exposed_names:
            self.exposed_consumer[exposed_name] = yield self.consume(
                '%s.outbound' % (exposed_name, ),
                functools.partial(self.dispatch_outbound_message,
                                  exposed_name),
                message_class=TransportUserMessage,
                paused=True,
                prefetch_count=self.amqp_prefetch_count)

    def dispatch_inbound_message(self, endpoint, msg):
        d = self._middlewares.apply_consume("inbound", msg, endpoint)
        d.addCallback(self._router.dispatch_inbound_message)
        return d

    def dispatch_inbound_event(self, endpoint, msg):
        d = self._middlewares.apply_consume("event", msg, endpoint)
        d.addCallback(self._router.dispatch_inbound_event)
        return d

    def dispatch_outbound_message(self, endpoint, msg):
        d = self._middlewares.apply_consume("outbound", msg, endpoint)
        d.addCallback(self._router.dispatch_outbound_message)
        return d

    def publish_inbound_message(self, endpoint, msg):
        d = self._middlewares.apply_publish("inbound", msg, endpoint)
        d.addCallback(self.exposed_publisher[endpoint].publish_message)
        return d

    def publish_inbound_event(self, endpoint, msg):
        d = self._middlewares.apply_publish("event", msg, endpoint)
        d.addCallback(self.exposed_event_publisher[endpoint].publish_message)
        return d

    def publish_outbound_message(self, endpoint, msg):
        d = self._middlewares.apply_publish("outbound", msg, endpoint)
        d.addCallback(self.transport_publisher[endpoint].publish_message)
        return d
Esempio n. 12
0
 def setup_middleware(self):
     middlewares = yield setup_middlewares_from_config(self, self.config)
     self._middlewares = MiddlewareStack(middlewares)
Esempio n. 13
0
class BaseDispatchWorker(Worker):
    """Base class for a dispatch worker.

    """

    @inlineCallbacks
    def startWorker(self):
        log.msg('Starting a %s dispatcher with config: %s'
                % (self.__class__.__name__, self.config))

        self.amqp_prefetch_count = self.config.get('amqp_prefetch_count', 20)
        yield self.setup_endpoints()
        yield self.setup_middleware()
        yield self.setup_router()
        yield self.setup_transport_publishers()
        yield self.setup_exposed_publishers()
        yield self.setup_transport_consumers()
        yield self.setup_exposed_consumers()

        consumers = (self.exposed_consumer.values() +
                        self.transport_consumer.values() +
                        self.transport_event_consumer.values())
        for consumer in consumers:
            consumer.unpause()

    @inlineCallbacks
    def stopWorker(self):
        yield self.teardown_router()
        yield self.teardown_middleware()

    def setup_endpoints(self):
        self.transport_names = self.config.get('transport_names', [])
        self.exposed_names = self.config.get('exposed_names', [])

    @inlineCallbacks
    def setup_middleware(self):
        middlewares = yield setup_middlewares_from_config(self, self.config)
        self._middlewares = MiddlewareStack(middlewares)

    def teardown_middleware(self):
        return self._middlewares.teardown()

    def setup_router(self):
        router_cls = load_class_by_string(self.config['router_class'])
        self._router = router_cls(self, self.config)
        return maybeDeferred(self._router.setup_routing)

    def teardown_router(self):
        return maybeDeferred(self._router.teardown_routing)

    @inlineCallbacks
    def setup_transport_publishers(self):
        self.transport_publisher = {}
        for transport_name in self.transport_names:
            self.transport_publisher[transport_name] = yield self.publish_to(
                '%s.outbound' % (transport_name,))

    @inlineCallbacks
    def setup_transport_consumers(self):
        self.transport_consumer = {}
        self.transport_event_consumer = {}
        for transport_name in self.transport_names:
            self.transport_consumer[transport_name] = yield self.consume(
                '%s.inbound' % (transport_name,),
                functools.partial(self.dispatch_inbound_message,
                                  transport_name),
                message_class=TransportUserMessage, paused=True,
                prefetch_count=self.amqp_prefetch_count)
        for transport_name in self.transport_names:
            self.transport_event_consumer[transport_name] = yield self.consume(
                '%s.event' % (transport_name,),
                functools.partial(self.dispatch_inbound_event, transport_name),
                message_class=TransportEvent, paused=True,
                prefetch_count=self.amqp_prefetch_count)

    @inlineCallbacks
    def setup_exposed_publishers(self):
        self.exposed_publisher = {}
        self.exposed_event_publisher = {}
        for exposed_name in self.exposed_names:
            self.exposed_publisher[exposed_name] = yield self.publish_to(
                '%s.inbound' % (exposed_name,))
        for exposed_name in self.exposed_names:
            self.exposed_event_publisher[exposed_name] = yield self.publish_to(
                '%s.event' % (exposed_name,))

    @inlineCallbacks
    def setup_exposed_consumers(self):
        self.exposed_consumer = {}
        for exposed_name in self.exposed_names:
            self.exposed_consumer[exposed_name] = yield self.consume(
                '%s.outbound' % (exposed_name,),
                functools.partial(self.dispatch_outbound_message,
                                  exposed_name),
                message_class=TransportUserMessage, paused=True,
                prefetch_count=self.amqp_prefetch_count)

    def dispatch_inbound_message(self, endpoint, msg):
        d = self._middlewares.apply_consume("inbound", msg, endpoint)
        d.addCallback(self._router.dispatch_inbound_message)
        return d

    def dispatch_inbound_event(self, endpoint, msg):
        d = self._middlewares.apply_consume("event", msg, endpoint)
        d.addCallback(self._router.dispatch_inbound_event)
        return d

    def dispatch_outbound_message(self, endpoint, msg):
        d = self._middlewares.apply_consume("outbound", msg, endpoint)
        d.addCallback(self._router.dispatch_outbound_message)
        return d

    def publish_inbound_message(self, endpoint, msg):
        d = self._middlewares.apply_publish("inbound", msg, endpoint)
        d.addCallback(self.exposed_publisher[endpoint].publish_message)
        return d

    def publish_inbound_event(self, endpoint, msg):
        d = self._middlewares.apply_publish("event", msg, endpoint)
        d.addCallback(self.exposed_event_publisher[endpoint].publish_message)
        return d

    def publish_outbound_message(self, endpoint, msg):
        d = self._middlewares.apply_publish("outbound", msg, endpoint)
        d.addCallback(self.transport_publisher[endpoint].publish_message)
        return d
Esempio n. 14
0
class BaseConnector(object):
    """Base class for 'connector' objects.

    A connector encapsulates the 'inbound', 'outbound' and 'event' publishers
    and consumers required by vumi workers and avoids having to operate on them
    individually all over the place.
    """
    def __init__(self,
                 worker,
                 connector_name,
                 prefetch_count=None,
                 middlewares=None):
        self.name = connector_name
        self.worker = worker
        self._consumers = {}
        self._publishers = {}
        self._endpoint_handlers = {}
        self._default_handlers = {}
        self._prefetch_count = prefetch_count
        self._middlewares = MiddlewareStack(
            middlewares if middlewares is not None else [])

    def _rkey(self, mtype):
        return '%s.%s' % (self.name, mtype)

    def setup(self):
        raise NotImplementedError()

    def teardown(self):
        d = gatherResults([c.stop() for c in self._consumers.values()])
        d.addCallback(lambda r: self._middlewares.teardown())
        return d

    @property
    def paused(self):
        return all(consumer.paused
                   for consumer in self._consumers.itervalues())

    def pause(self):
        return gatherResults(
            [consumer.pause() for consumer in self._consumers.itervalues()])

    def unpause(self):
        # This doesn't return a deferred.
        for consumer in self._consumers.values():
            consumer.unpause()

    @inlineCallbacks
    def _setup_publisher(self, mtype):
        publisher = yield self.worker.publish_to(self._rkey(mtype))
        self._publishers[mtype] = publisher
        returnValue(publisher)

    @inlineCallbacks
    def _setup_consumer(self, mtype, msg_class, default_handler):
        def handler(msg):
            return self._consume_message(mtype, msg)

        consumer = yield self.worker.consume(
            self._rkey(mtype),
            handler,
            message_class=msg_class,
            paused=True,
            prefetch_count=self._prefetch_count)
        self._consumers[mtype] = consumer
        self._set_default_endpoint_handler(mtype, default_handler)
        returnValue(consumer)

    def _set_endpoint_handler(self, mtype, handler, endpoint_name):
        if endpoint_name is None:
            endpoint_name = TransportMessage.DEFAULT_ENDPOINT_NAME
        handlers = self._endpoint_handlers.setdefault(mtype, {})
        handlers[endpoint_name] = handler

    def _set_default_endpoint_handler(self, mtype, handler):
        self._endpoint_handlers.setdefault(mtype, {})
        self._default_handlers[mtype] = handler

    def _consume_message(self, mtype, msg):
        endpoint_name = msg.get_routing_endpoint()
        handler = self._endpoint_handlers[mtype].get(endpoint_name)
        if handler is None:
            handler = self._default_handlers.get(mtype)
        d = self._middlewares.apply_consume(mtype, msg, self.name)
        d.addCallback(handler)
        return d.addErrback(self._ignore_message, msg)

    def _publish_message(self, mtype, msg, endpoint_name):
        if endpoint_name is not None:
            msg.set_routing_endpoint(endpoint_name)
        d = self._middlewares.apply_publish(mtype, msg, self.name)
        return d.addCallback(self._publishers[mtype].publish_message)

    def _ignore_message(self, failure, msg):
        failure.trap(IgnoreMessage)
        log.debug("Ignoring msg due to %r: %r" % (failure.value, msg))
Esempio n. 15
0
class Transport(Worker):
    """
    Base class for transport workers.

    The following attributes are available for subclasses to control behaviour:

    * :attr:`start_message_consumer` -- Set to ``False`` if the message
      consumer should not be started. The subclass is responsible for starting
      it in this case.
    """

    SUPPRESS_FAILURE_EXCEPTIONS = True

    transport_name = None
    start_message_consumer = True

    @inlineCallbacks
    def startWorker(self):
        """
        Set up basic transport worker stuff.

        You shouldn't have to override this in subclasses.
        """
        self._consumers = []

        self._validate_config()
        if 'TRANSPORT_NAME' in self.config:
            log.msg("NOTE: 'TRANSPORT_NAME' in config is deprecated. "
                    "Use 'transport_name' instead.")
            self.config.setdefault('transport_name',
                                   self.config['TRANSPORT_NAME'])
        self.transport_name = self.config['transport_name']
        self.concurrent_sends = self.config.get('concurrent_sends')

        yield self._setup_failure_publisher()
        yield self._setup_message_publisher()
        yield self._setup_event_publisher()

        yield self.setup_middleware()

        yield self.setup_transport()

        self.message_consumer = None
        if self.start_message_consumer:
            yield self._setup_message_consumer()

    @inlineCallbacks
    def stopWorker(self):
        while self._consumers:
            consumer = self._consumers.pop()
            yield consumer.stop()
        yield self.teardown_transport()

    def get_rkey(self, mtype):
        return '%s.%s' % (self.transport_name, mtype)

    def publish_rkey(self, name):
        return self.publish_to(self.get_rkey(name))

    def _validate_config(self):
        if 'transport_name' not in self.config:
            raise ConfigError("Missing 'transport_name' field in config.")
        return self.validate_config()

    def validate_config(self):
        """
        Transport-specific config validation happens in here.

        Subclasses may override this method to perform extra config validation.
        """
        pass

    def setup_transport(self):
        """
        All transport_specific setup should happen in here.

        Subclasses should override this method to perform extra setup.
        """
        pass

    def teardown_transport(self):
        """
        Clean-up of setup done in setup_transport should happen here.
        """
        pass

    @inlineCallbacks
    def setup_middleware(self):
        """
        Middleware setup happens here.

        Subclasses should not override this unless they need to do nonstandard
        middleware setup.
        """
        middlewares = yield setup_middlewares_from_config(self, self.config)
        self._middlewares = MiddlewareStack(middlewares)

    @inlineCallbacks
    def _setup_message_publisher(self):
        self.message_publisher = yield self.publish_rkey('inbound')

    @inlineCallbacks
    def _setup_message_consumer(self):
        if self.message_consumer is not None:
            log.msg("Consumer already exists, not restarting.")
            return

        self.message_consumer = yield self.consume(
            self.get_rkey('outbound'), self._process_message,
            message_class=TransportUserMessage)
        self._consumers.append(self.message_consumer)

        # Apply concurrency throttling if we need to.
        if self.concurrent_sends is not None:
            yield self.message_consumer.channel.basic_qos(
                0, int(self.concurrent_sends), False)

    def _teardown_message_consumer(self):
        if self.message_consumer is None:
            log.msg("Consumer does not exist, not stopping.")
            return
        if self.message_consumer in self._consumers:
            self._consumers.remove(self.message_consumer)
        consumer, self.message_consumer = self.message_consumer, None
        return consumer.stop()

    @inlineCallbacks
    def _setup_event_publisher(self):
        self.event_publisher = yield self.publish_rkey('event')

    @inlineCallbacks
    def _setup_failure_publisher(self):
        self.failure_publisher = yield self.publish_rkey('failures')

    def send_failure(self, message, exception, traceback):
        """Send a failure report."""
        try:
            failure_code = getattr(exception, "failure_code",
                                   FailureMessage.FC_UNSPECIFIED)
            failure_msg = FailureMessage(
                    message=message.payload, failure_code=failure_code,
                    reason=traceback)
            d = self._middlewares.apply_publish("failure", failure_msg,
                                                self.transport_name)
            d.addCallback(self.failure_publisher.publish_message)
            d.addCallback(lambda _f: self.failure_published())
        except:
            log.err("Error publishing failure: %s, %s, %s"
                    % (message, exception, traceback))
            raise
        return d

    def failure_published(self):
        pass

    def publish_message(self, **kw):
        """
        Publish a :class:`TransportUserMessage` message.

        Some default parameters are handled, so subclasses don't have
        to provide a lot of boilerplate.
        """
        kw.setdefault('transport_name', self.transport_name)
        kw.setdefault('transport_metadata', {})
        msg = TransportUserMessage(**kw)
        d = self._middlewares.apply_publish("inbound", msg,
                                            self.transport_name)
        d.addCallback(self.message_publisher.publish_message)
        return d

    def publish_event(self, **kw):
        """
        Publish a :class:`TransportEvent` message.

        Some default parameters are handled, so subclasses don't have
        to provide a lot of boilerplate.
        """
        kw.setdefault('transport_name', self.transport_name)
        kw.setdefault('transport_metadata', {})
        event = TransportEvent(**kw)
        d = self._middlewares.apply_publish("event", event,
                                            self.transport_name)
        d.addCallback(self.event_publisher.publish_message)
        return d

    def publish_ack(self, user_message_id, sent_message_id, **kw):
        """
        Helper method for publishing an ``ack`` event.
        """
        return self.publish_event(user_message_id=user_message_id,
                                  sent_message_id=sent_message_id,
                                  event_type='ack', **kw)

    def publish_delivery_report(self, user_message_id, delivery_status, **kw):
        """
        Helper method for publishing a ``delivery_report`` event.
        """
        return self.publish_event(user_message_id=user_message_id,
                                  delivery_status=delivery_status,
                                  event_type='delivery_report', **kw)

    def _process_message(self, message):
        def _send_failure(f):
            self.send_failure(message, f.value, f.getTraceback())
            log.err(f)
            if self.SUPPRESS_FAILURE_EXCEPTIONS:
                return None
            return f

        d = self._middlewares.apply_consume("outbound", message,
                                            self.transport_name)
        d.addCallback(self.handle_outbound_message)
        d.addErrback(_send_failure)
        return d

    def handle_outbound_message(self, message):
        """
        This must be overridden to read outbound messages and do the right
        thing with them.
        """
        raise NotImplementedError()

    @staticmethod
    def generate_message_id():
        """
        Generate a message id.
        """

        return TransportUserMessage.generate_id()