示例#1
0
class BaseConnector(object):
    """Base class for 'connector' objects.

    A connector encapsulates the 'inbound', 'outbound' and 'event' publishers
    and consumers required by vumi workers and avoids having to operate on them
    individually all over the place.
    """
    def __init__(self, worker, connector_name, prefetch_count=None,
                 middlewares=None):
        self.name = connector_name
        self.worker = worker
        self._consumers = {}
        self._publishers = {}
        self._endpoint_handlers = {}
        self._default_handlers = {}
        self._prefetch_count = prefetch_count
        self._middlewares = MiddlewareStack(middlewares
                                            if middlewares is not None else [])

    def _rkey(self, mtype):
        return '%s.%s' % (self.name, mtype)

    def setup(self):
        raise NotImplementedError()

    def teardown(self):
        d = gatherResults([c.stop() for c in self._consumers.values()])
        d.addCallback(lambda r: self._middlewares.teardown())
        return d

    @property
    def paused(self):
        return all(consumer.paused
                   for consumer in self._consumers.itervalues())

    def pause(self):
        return gatherResults([
            consumer.pause() for consumer in self._consumers.itervalues()])

    def unpause(self):
        # This doesn't return a deferred.
        for consumer in self._consumers.values():
            consumer.unpause()

    @inlineCallbacks
    def _setup_publisher(self, mtype):
        publisher = yield self.worker.publish_to(self._rkey(mtype))
        self._publishers[mtype] = publisher
        returnValue(publisher)

    @inlineCallbacks
    def _setup_consumer(self, mtype, msg_class, default_handler):
        def handler(msg):
            return self._consume_message(mtype, msg)

        consumer = yield self.worker.consume(
            self._rkey(mtype), handler, message_class=msg_class, paused=True,
            prefetch_count=self._prefetch_count)
        self._consumers[mtype] = consumer
        self._set_default_endpoint_handler(mtype, default_handler)
        returnValue(consumer)

    def _set_endpoint_handler(self, mtype, handler, endpoint_name):
        if endpoint_name is None:
            endpoint_name = TransportMessage.DEFAULT_ENDPOINT_NAME
        handlers = self._endpoint_handlers.setdefault(mtype, {})
        handlers[endpoint_name] = handler

    def _set_default_endpoint_handler(self, mtype, handler):
        self._endpoint_handlers.setdefault(mtype, {})
        self._default_handlers[mtype] = handler

    def _consume_message(self, mtype, msg):
        endpoint_name = msg.get_routing_endpoint()
        handler = self._endpoint_handlers[mtype].get(endpoint_name)
        if handler is None:
            handler = self._default_handlers.get(mtype)
        d = self._middlewares.apply_consume(mtype, msg, self.name)
        d.addCallback(handler)
        return d.addErrback(self._ignore_message, msg)

    def _publish_message(self, mtype, msg, endpoint_name):
        if endpoint_name is not None:
            msg.set_routing_endpoint(endpoint_name)
        d = self._middlewares.apply_publish(mtype, msg, self.name)
        return d.addCallback(self._publishers[mtype].publish_message)

    def _ignore_message(self, failure, msg):
        failure.trap(IgnoreMessage)
        log.debug("Ignoring msg due to %r: %r" % (failure.value, msg))
示例#2
0
文件: base.py 项目: AndrewCvekl/vumi
class BaseDispatchWorker(Worker):
    """Base class for a dispatch worker.

    """

    @inlineCallbacks
    def startWorker(self):
        log.msg('Starting a %s dispatcher with config: %s'
                % (self.__class__.__name__, self.config))

        self.amqp_prefetch_count = self.config.get('amqp_prefetch_count', 20)
        yield self.setup_endpoints()
        yield self.setup_middleware()
        yield self.setup_router()
        yield self.setup_transport_publishers()
        yield self.setup_exposed_publishers()
        yield self.setup_transport_consumers()
        yield self.setup_exposed_consumers()

        consumers = (self.exposed_consumer.values() +
                        self.transport_consumer.values() +
                        self.transport_event_consumer.values())
        for consumer in consumers:
            consumer.unpause()

    @inlineCallbacks
    def stopWorker(self):
        yield self.teardown_router()
        yield self.teardown_middleware()

    def setup_endpoints(self):
        self.transport_names = self.config.get('transport_names', [])
        self.exposed_names = self.config.get('exposed_names', [])

    @inlineCallbacks
    def setup_middleware(self):
        middlewares = yield setup_middlewares_from_config(self, self.config)
        self._middlewares = MiddlewareStack(middlewares)

    def teardown_middleware(self):
        return self._middlewares.teardown()

    def setup_router(self):
        router_cls = load_class_by_string(self.config['router_class'])
        self._router = router_cls(self, self.config)
        return maybeDeferred(self._router.setup_routing)

    def teardown_router(self):
        return maybeDeferred(self._router.teardown_routing)

    @inlineCallbacks
    def setup_transport_publishers(self):
        self.transport_publisher = {}
        for transport_name in self.transport_names:
            self.transport_publisher[transport_name] = yield self.publish_to(
                '%s.outbound' % (transport_name,))

    @inlineCallbacks
    def setup_transport_consumers(self):
        self.transport_consumer = {}
        self.transport_event_consumer = {}
        for transport_name in self.transport_names:
            self.transport_consumer[transport_name] = yield self.consume(
                '%s.inbound' % (transport_name,),
                functools.partial(self.dispatch_inbound_message,
                                  transport_name),
                message_class=TransportUserMessage, paused=True,
                prefetch_count=self.amqp_prefetch_count)
        for transport_name in self.transport_names:
            self.transport_event_consumer[transport_name] = yield self.consume(
                '%s.event' % (transport_name,),
                functools.partial(self.dispatch_inbound_event, transport_name),
                message_class=TransportEvent, paused=True,
                prefetch_count=self.amqp_prefetch_count)

    @inlineCallbacks
    def setup_exposed_publishers(self):
        self.exposed_publisher = {}
        self.exposed_event_publisher = {}
        for exposed_name in self.exposed_names:
            self.exposed_publisher[exposed_name] = yield self.publish_to(
                '%s.inbound' % (exposed_name,))
        for exposed_name in self.exposed_names:
            self.exposed_event_publisher[exposed_name] = yield self.publish_to(
                '%s.event' % (exposed_name,))

    @inlineCallbacks
    def setup_exposed_consumers(self):
        self.exposed_consumer = {}
        for exposed_name in self.exposed_names:
            self.exposed_consumer[exposed_name] = yield self.consume(
                '%s.outbound' % (exposed_name,),
                functools.partial(self.dispatch_outbound_message,
                                  exposed_name),
                message_class=TransportUserMessage, paused=True,
                prefetch_count=self.amqp_prefetch_count)

    def dispatch_inbound_message(self, endpoint, msg):
        d = self._middlewares.apply_consume("inbound", msg, endpoint)
        d.addCallback(self._router.dispatch_inbound_message)
        return d

    def dispatch_inbound_event(self, endpoint, msg):
        d = self._middlewares.apply_consume("event", msg, endpoint)
        d.addCallback(self._router.dispatch_inbound_event)
        return d

    def dispatch_outbound_message(self, endpoint, msg):
        d = self._middlewares.apply_consume("outbound", msg, endpoint)
        d.addCallback(self._router.dispatch_outbound_message)
        return d

    def publish_inbound_message(self, endpoint, msg):
        d = self._middlewares.apply_publish("inbound", msg, endpoint)
        d.addCallback(self.exposed_publisher[endpoint].publish_message)
        return d

    def publish_inbound_event(self, endpoint, msg):
        d = self._middlewares.apply_publish("event", msg, endpoint)
        d.addCallback(self.exposed_event_publisher[endpoint].publish_message)
        return d

    def publish_outbound_message(self, endpoint, msg):
        d = self._middlewares.apply_publish("outbound", msg, endpoint)
        d.addCallback(self.transport_publisher[endpoint].publish_message)
        return d
示例#3
0
class BaseDispatchWorker(Worker):
    """Base class for a dispatch worker.

    """
    @inlineCallbacks
    def startWorker(self):
        log.msg('Starting a %s dispatcher with config: %s' %
                (self.__class__.__name__, self.config))

        self.amqp_prefetch_count = self.config.get('amqp_prefetch_count', 20)
        yield self.setup_endpoints()
        yield self.setup_middleware()
        yield self.setup_router()
        yield self.setup_transport_publishers()
        yield self.setup_exposed_publishers()
        yield self.setup_transport_consumers()
        yield self.setup_exposed_consumers()

        consumers = (self.exposed_consumer.values() +
                     self.transport_consumer.values() +
                     self.transport_event_consumer.values())
        for consumer in consumers:
            consumer.unpause()

    @inlineCallbacks
    def stopWorker(self):
        yield self.teardown_router()
        yield self.teardown_middleware()

    def setup_endpoints(self):
        self.transport_names = self.config.get('transport_names', [])
        self.exposed_names = self.config.get('exposed_names', [])

    @inlineCallbacks
    def setup_middleware(self):
        middlewares = yield setup_middlewares_from_config(self, self.config)
        self._middlewares = MiddlewareStack(middlewares)

    def teardown_middleware(self):
        return self._middlewares.teardown()

    def setup_router(self):
        router_cls = load_class_by_string(self.config['router_class'])
        self._router = router_cls(self, self.config)
        return maybeDeferred(self._router.setup_routing)

    def teardown_router(self):
        return maybeDeferred(self._router.teardown_routing)

    @inlineCallbacks
    def setup_transport_publishers(self):
        self.transport_publisher = {}
        for transport_name in self.transport_names:
            self.transport_publisher[transport_name] = yield self.publish_to(
                '%s.outbound' % (transport_name, ))

    @inlineCallbacks
    def setup_transport_consumers(self):
        self.transport_consumer = {}
        self.transport_event_consumer = {}
        for transport_name in self.transport_names:
            self.transport_consumer[transport_name] = yield self.consume(
                '%s.inbound' % (transport_name, ),
                functools.partial(self.dispatch_inbound_message,
                                  transport_name),
                message_class=TransportUserMessage,
                paused=True,
                prefetch_count=self.amqp_prefetch_count)
        for transport_name in self.transport_names:
            self.transport_event_consumer[transport_name] = yield self.consume(
                '%s.event' % (transport_name, ),
                functools.partial(self.dispatch_inbound_event, transport_name),
                message_class=TransportEvent,
                paused=True,
                prefetch_count=self.amqp_prefetch_count)

    @inlineCallbacks
    def setup_exposed_publishers(self):
        self.exposed_publisher = {}
        self.exposed_event_publisher = {}
        for exposed_name in self.exposed_names:
            self.exposed_publisher[exposed_name] = yield self.publish_to(
                '%s.inbound' % (exposed_name, ))
        for exposed_name in self.exposed_names:
            self.exposed_event_publisher[exposed_name] = yield self.publish_to(
                '%s.event' % (exposed_name, ))

    @inlineCallbacks
    def setup_exposed_consumers(self):
        self.exposed_consumer = {}
        for exposed_name in self.exposed_names:
            self.exposed_consumer[exposed_name] = yield self.consume(
                '%s.outbound' % (exposed_name, ),
                functools.partial(self.dispatch_outbound_message,
                                  exposed_name),
                message_class=TransportUserMessage,
                paused=True,
                prefetch_count=self.amqp_prefetch_count)

    def dispatch_inbound_message(self, endpoint, msg):
        d = self._middlewares.apply_consume("inbound", msg, endpoint)
        d.addCallback(self._router.dispatch_inbound_message)
        return d

    def dispatch_inbound_event(self, endpoint, msg):
        d = self._middlewares.apply_consume("event", msg, endpoint)
        d.addCallback(self._router.dispatch_inbound_event)
        return d

    def dispatch_outbound_message(self, endpoint, msg):
        d = self._middlewares.apply_consume("outbound", msg, endpoint)
        d.addCallback(self._router.dispatch_outbound_message)
        return d

    def publish_inbound_message(self, endpoint, msg):
        d = self._middlewares.apply_publish("inbound", msg, endpoint)
        d.addCallback(self.exposed_publisher[endpoint].publish_message)
        return d

    def publish_inbound_event(self, endpoint, msg):
        d = self._middlewares.apply_publish("event", msg, endpoint)
        d.addCallback(self.exposed_event_publisher[endpoint].publish_message)
        return d

    def publish_outbound_message(self, endpoint, msg):
        d = self._middlewares.apply_publish("outbound", msg, endpoint)
        d.addCallback(self.transport_publisher[endpoint].publish_message)
        return d
示例#4
0
class BaseConnector(object):
    """Base class for 'connector' objects.

    A connector encapsulates the 'inbound', 'outbound' and 'event' publishers
    and consumers required by vumi workers and avoids having to operate on them
    individually all over the place.
    """
    def __init__(self,
                 worker,
                 connector_name,
                 prefetch_count=None,
                 middlewares=None):
        self.name = connector_name
        self.worker = worker
        self._consumers = {}
        self._publishers = {}
        self._endpoint_handlers = {}
        self._default_handlers = {}
        self._prefetch_count = prefetch_count
        self._middlewares = MiddlewareStack(
            middlewares if middlewares is not None else [])

    def _rkey(self, mtype):
        return '%s.%s' % (self.name, mtype)

    def setup(self):
        raise NotImplementedError()

    def teardown(self):
        d = gatherResults([c.stop() for c in self._consumers.values()])
        d.addCallback(lambda r: self._middlewares.teardown())
        return d

    @property
    def paused(self):
        return all(consumer.paused
                   for consumer in self._consumers.itervalues())

    def pause(self):
        return gatherResults(
            [consumer.pause() for consumer in self._consumers.itervalues()])

    def unpause(self):
        # This doesn't return a deferred.
        for consumer in self._consumers.values():
            consumer.unpause()

    @inlineCallbacks
    def _setup_publisher(self, mtype):
        publisher = yield self.worker.publish_to(self._rkey(mtype))
        self._publishers[mtype] = publisher
        returnValue(publisher)

    @inlineCallbacks
    def _setup_consumer(self, mtype, msg_class, default_handler):
        def handler(msg):
            return self._consume_message(mtype, msg)

        consumer = yield self.worker.consume(
            self._rkey(mtype),
            handler,
            message_class=msg_class,
            paused=True,
            prefetch_count=self._prefetch_count)
        self._consumers[mtype] = consumer
        self._set_default_endpoint_handler(mtype, default_handler)
        returnValue(consumer)

    def _set_endpoint_handler(self, mtype, handler, endpoint_name):
        if endpoint_name is None:
            endpoint_name = TransportMessage.DEFAULT_ENDPOINT_NAME
        handlers = self._endpoint_handlers.setdefault(mtype, {})
        handlers[endpoint_name] = handler

    def _set_default_endpoint_handler(self, mtype, handler):
        self._endpoint_handlers.setdefault(mtype, {})
        self._default_handlers[mtype] = handler

    def _consume_message(self, mtype, msg):
        endpoint_name = msg.get_routing_endpoint()
        handler = self._endpoint_handlers[mtype].get(endpoint_name)
        if handler is None:
            handler = self._default_handlers.get(mtype)
        d = self._middlewares.apply_consume(mtype, msg, self.name)
        d.addCallback(handler)
        return d.addErrback(self._ignore_message, msg)

    def _publish_message(self, mtype, msg, endpoint_name):
        if endpoint_name is not None:
            msg.set_routing_endpoint(endpoint_name)
        d = self._middlewares.apply_publish(mtype, msg, self.name)
        return d.addCallback(self._publishers[mtype].publish_message)

    def _ignore_message(self, failure, msg):
        failure.trap(IgnoreMessage)
        log.debug("Ignoring msg due to %r: %r" % (failure.value, msg))