class BaseConnector(object): """Base class for 'connector' objects. A connector encapsulates the 'inbound', 'outbound' and 'event' publishers and consumers required by vumi workers and avoids having to operate on them individually all over the place. """ def __init__(self, worker, connector_name, prefetch_count=None, middlewares=None): self.name = connector_name self.worker = worker self._consumers = {} self._publishers = {} self._endpoint_handlers = {} self._default_handlers = {} self._prefetch_count = prefetch_count self._middlewares = MiddlewareStack(middlewares if middlewares is not None else []) def _rkey(self, mtype): return '%s.%s' % (self.name, mtype) def setup(self): raise NotImplementedError() def teardown(self): d = gatherResults([c.stop() for c in self._consumers.values()]) d.addCallback(lambda r: self._middlewares.teardown()) return d @property def paused(self): return all(consumer.paused for consumer in self._consumers.itervalues()) def pause(self): return gatherResults([ consumer.pause() for consumer in self._consumers.itervalues()]) def unpause(self): # This doesn't return a deferred. for consumer in self._consumers.values(): consumer.unpause() @inlineCallbacks def _setup_publisher(self, mtype): publisher = yield self.worker.publish_to(self._rkey(mtype)) self._publishers[mtype] = publisher returnValue(publisher) @inlineCallbacks def _setup_consumer(self, mtype, msg_class, default_handler): def handler(msg): return self._consume_message(mtype, msg) consumer = yield self.worker.consume( self._rkey(mtype), handler, message_class=msg_class, paused=True, prefetch_count=self._prefetch_count) self._consumers[mtype] = consumer self._set_default_endpoint_handler(mtype, default_handler) returnValue(consumer) def _set_endpoint_handler(self, mtype, handler, endpoint_name): if endpoint_name is None: endpoint_name = TransportMessage.DEFAULT_ENDPOINT_NAME handlers = self._endpoint_handlers.setdefault(mtype, {}) handlers[endpoint_name] = handler def _set_default_endpoint_handler(self, mtype, handler): self._endpoint_handlers.setdefault(mtype, {}) self._default_handlers[mtype] = handler def _consume_message(self, mtype, msg): endpoint_name = msg.get_routing_endpoint() handler = self._endpoint_handlers[mtype].get(endpoint_name) if handler is None: handler = self._default_handlers.get(mtype) d = self._middlewares.apply_consume(mtype, msg, self.name) d.addCallback(handler) return d.addErrback(self._ignore_message, msg) def _publish_message(self, mtype, msg, endpoint_name): if endpoint_name is not None: msg.set_routing_endpoint(endpoint_name) d = self._middlewares.apply_publish(mtype, msg, self.name) return d.addCallback(self._publishers[mtype].publish_message) def _ignore_message(self, failure, msg): failure.trap(IgnoreMessage) log.debug("Ignoring msg due to %r: %r" % (failure.value, msg))
class BaseDispatchWorker(Worker): """Base class for a dispatch worker. """ @inlineCallbacks def startWorker(self): log.msg('Starting a %s dispatcher with config: %s' % (self.__class__.__name__, self.config)) self.amqp_prefetch_count = self.config.get('amqp_prefetch_count', 20) yield self.setup_endpoints() yield self.setup_middleware() yield self.setup_router() yield self.setup_transport_publishers() yield self.setup_exposed_publishers() yield self.setup_transport_consumers() yield self.setup_exposed_consumers() consumers = (self.exposed_consumer.values() + self.transport_consumer.values() + self.transport_event_consumer.values()) for consumer in consumers: consumer.unpause() @inlineCallbacks def stopWorker(self): yield self.teardown_router() yield self.teardown_middleware() def setup_endpoints(self): self.transport_names = self.config.get('transport_names', []) self.exposed_names = self.config.get('exposed_names', []) @inlineCallbacks def setup_middleware(self): middlewares = yield setup_middlewares_from_config(self, self.config) self._middlewares = MiddlewareStack(middlewares) def teardown_middleware(self): return self._middlewares.teardown() def setup_router(self): router_cls = load_class_by_string(self.config['router_class']) self._router = router_cls(self, self.config) return maybeDeferred(self._router.setup_routing) def teardown_router(self): return maybeDeferred(self._router.teardown_routing) @inlineCallbacks def setup_transport_publishers(self): self.transport_publisher = {} for transport_name in self.transport_names: self.transport_publisher[transport_name] = yield self.publish_to( '%s.outbound' % (transport_name,)) @inlineCallbacks def setup_transport_consumers(self): self.transport_consumer = {} self.transport_event_consumer = {} for transport_name in self.transport_names: self.transport_consumer[transport_name] = yield self.consume( '%s.inbound' % (transport_name,), functools.partial(self.dispatch_inbound_message, transport_name), message_class=TransportUserMessage, paused=True, prefetch_count=self.amqp_prefetch_count) for transport_name in self.transport_names: self.transport_event_consumer[transport_name] = yield self.consume( '%s.event' % (transport_name,), functools.partial(self.dispatch_inbound_event, transport_name), message_class=TransportEvent, paused=True, prefetch_count=self.amqp_prefetch_count) @inlineCallbacks def setup_exposed_publishers(self): self.exposed_publisher = {} self.exposed_event_publisher = {} for exposed_name in self.exposed_names: self.exposed_publisher[exposed_name] = yield self.publish_to( '%s.inbound' % (exposed_name,)) for exposed_name in self.exposed_names: self.exposed_event_publisher[exposed_name] = yield self.publish_to( '%s.event' % (exposed_name,)) @inlineCallbacks def setup_exposed_consumers(self): self.exposed_consumer = {} for exposed_name in self.exposed_names: self.exposed_consumer[exposed_name] = yield self.consume( '%s.outbound' % (exposed_name,), functools.partial(self.dispatch_outbound_message, exposed_name), message_class=TransportUserMessage, paused=True, prefetch_count=self.amqp_prefetch_count) def dispatch_inbound_message(self, endpoint, msg): d = self._middlewares.apply_consume("inbound", msg, endpoint) d.addCallback(self._router.dispatch_inbound_message) return d def dispatch_inbound_event(self, endpoint, msg): d = self._middlewares.apply_consume("event", msg, endpoint) d.addCallback(self._router.dispatch_inbound_event) return d def dispatch_outbound_message(self, endpoint, msg): d = self._middlewares.apply_consume("outbound", msg, endpoint) d.addCallback(self._router.dispatch_outbound_message) return d def publish_inbound_message(self, endpoint, msg): d = self._middlewares.apply_publish("inbound", msg, endpoint) d.addCallback(self.exposed_publisher[endpoint].publish_message) return d def publish_inbound_event(self, endpoint, msg): d = self._middlewares.apply_publish("event", msg, endpoint) d.addCallback(self.exposed_event_publisher[endpoint].publish_message) return d def publish_outbound_message(self, endpoint, msg): d = self._middlewares.apply_publish("outbound", msg, endpoint) d.addCallback(self.transport_publisher[endpoint].publish_message) return d
class BaseDispatchWorker(Worker): """Base class for a dispatch worker. """ @inlineCallbacks def startWorker(self): log.msg('Starting a %s dispatcher with config: %s' % (self.__class__.__name__, self.config)) self.amqp_prefetch_count = self.config.get('amqp_prefetch_count', 20) yield self.setup_endpoints() yield self.setup_middleware() yield self.setup_router() yield self.setup_transport_publishers() yield self.setup_exposed_publishers() yield self.setup_transport_consumers() yield self.setup_exposed_consumers() consumers = (self.exposed_consumer.values() + self.transport_consumer.values() + self.transport_event_consumer.values()) for consumer in consumers: consumer.unpause() @inlineCallbacks def stopWorker(self): yield self.teardown_router() yield self.teardown_middleware() def setup_endpoints(self): self.transport_names = self.config.get('transport_names', []) self.exposed_names = self.config.get('exposed_names', []) @inlineCallbacks def setup_middleware(self): middlewares = yield setup_middlewares_from_config(self, self.config) self._middlewares = MiddlewareStack(middlewares) def teardown_middleware(self): return self._middlewares.teardown() def setup_router(self): router_cls = load_class_by_string(self.config['router_class']) self._router = router_cls(self, self.config) return maybeDeferred(self._router.setup_routing) def teardown_router(self): return maybeDeferred(self._router.teardown_routing) @inlineCallbacks def setup_transport_publishers(self): self.transport_publisher = {} for transport_name in self.transport_names: self.transport_publisher[transport_name] = yield self.publish_to( '%s.outbound' % (transport_name, )) @inlineCallbacks def setup_transport_consumers(self): self.transport_consumer = {} self.transport_event_consumer = {} for transport_name in self.transport_names: self.transport_consumer[transport_name] = yield self.consume( '%s.inbound' % (transport_name, ), functools.partial(self.dispatch_inbound_message, transport_name), message_class=TransportUserMessage, paused=True, prefetch_count=self.amqp_prefetch_count) for transport_name in self.transport_names: self.transport_event_consumer[transport_name] = yield self.consume( '%s.event' % (transport_name, ), functools.partial(self.dispatch_inbound_event, transport_name), message_class=TransportEvent, paused=True, prefetch_count=self.amqp_prefetch_count) @inlineCallbacks def setup_exposed_publishers(self): self.exposed_publisher = {} self.exposed_event_publisher = {} for exposed_name in self.exposed_names: self.exposed_publisher[exposed_name] = yield self.publish_to( '%s.inbound' % (exposed_name, )) for exposed_name in self.exposed_names: self.exposed_event_publisher[exposed_name] = yield self.publish_to( '%s.event' % (exposed_name, )) @inlineCallbacks def setup_exposed_consumers(self): self.exposed_consumer = {} for exposed_name in self.exposed_names: self.exposed_consumer[exposed_name] = yield self.consume( '%s.outbound' % (exposed_name, ), functools.partial(self.dispatch_outbound_message, exposed_name), message_class=TransportUserMessage, paused=True, prefetch_count=self.amqp_prefetch_count) def dispatch_inbound_message(self, endpoint, msg): d = self._middlewares.apply_consume("inbound", msg, endpoint) d.addCallback(self._router.dispatch_inbound_message) return d def dispatch_inbound_event(self, endpoint, msg): d = self._middlewares.apply_consume("event", msg, endpoint) d.addCallback(self._router.dispatch_inbound_event) return d def dispatch_outbound_message(self, endpoint, msg): d = self._middlewares.apply_consume("outbound", msg, endpoint) d.addCallback(self._router.dispatch_outbound_message) return d def publish_inbound_message(self, endpoint, msg): d = self._middlewares.apply_publish("inbound", msg, endpoint) d.addCallback(self.exposed_publisher[endpoint].publish_message) return d def publish_inbound_event(self, endpoint, msg): d = self._middlewares.apply_publish("event", msg, endpoint) d.addCallback(self.exposed_event_publisher[endpoint].publish_message) return d def publish_outbound_message(self, endpoint, msg): d = self._middlewares.apply_publish("outbound", msg, endpoint) d.addCallback(self.transport_publisher[endpoint].publish_message) return d
class BaseConnector(object): """Base class for 'connector' objects. A connector encapsulates the 'inbound', 'outbound' and 'event' publishers and consumers required by vumi workers and avoids having to operate on them individually all over the place. """ def __init__(self, worker, connector_name, prefetch_count=None, middlewares=None): self.name = connector_name self.worker = worker self._consumers = {} self._publishers = {} self._endpoint_handlers = {} self._default_handlers = {} self._prefetch_count = prefetch_count self._middlewares = MiddlewareStack( middlewares if middlewares is not None else []) def _rkey(self, mtype): return '%s.%s' % (self.name, mtype) def setup(self): raise NotImplementedError() def teardown(self): d = gatherResults([c.stop() for c in self._consumers.values()]) d.addCallback(lambda r: self._middlewares.teardown()) return d @property def paused(self): return all(consumer.paused for consumer in self._consumers.itervalues()) def pause(self): return gatherResults( [consumer.pause() for consumer in self._consumers.itervalues()]) def unpause(self): # This doesn't return a deferred. for consumer in self._consumers.values(): consumer.unpause() @inlineCallbacks def _setup_publisher(self, mtype): publisher = yield self.worker.publish_to(self._rkey(mtype)) self._publishers[mtype] = publisher returnValue(publisher) @inlineCallbacks def _setup_consumer(self, mtype, msg_class, default_handler): def handler(msg): return self._consume_message(mtype, msg) consumer = yield self.worker.consume( self._rkey(mtype), handler, message_class=msg_class, paused=True, prefetch_count=self._prefetch_count) self._consumers[mtype] = consumer self._set_default_endpoint_handler(mtype, default_handler) returnValue(consumer) def _set_endpoint_handler(self, mtype, handler, endpoint_name): if endpoint_name is None: endpoint_name = TransportMessage.DEFAULT_ENDPOINT_NAME handlers = self._endpoint_handlers.setdefault(mtype, {}) handlers[endpoint_name] = handler def _set_default_endpoint_handler(self, mtype, handler): self._endpoint_handlers.setdefault(mtype, {}) self._default_handlers[mtype] = handler def _consume_message(self, mtype, msg): endpoint_name = msg.get_routing_endpoint() handler = self._endpoint_handlers[mtype].get(endpoint_name) if handler is None: handler = self._default_handlers.get(mtype) d = self._middlewares.apply_consume(mtype, msg, self.name) d.addCallback(handler) return d.addErrback(self._ignore_message, msg) def _publish_message(self, mtype, msg, endpoint_name): if endpoint_name is not None: msg.set_routing_endpoint(endpoint_name) d = self._middlewares.apply_publish(mtype, msg, self.name) return d.addCallback(self._publishers[mtype].publish_message) def _ignore_message(self, failure, msg): failure.trap(IgnoreMessage) log.debug("Ignoring msg due to %r: %r" % (failure.value, msg))