def setup_middleware(self):
"""
Middleware setup happens here.
Subclasses should not override this unless they need to do nonstandard
middleware setup.
"""
middlewares = yield setup_middlewares_from_config(self, self.config)
self._middlewares = MiddlewareStack(middlewares)
def __init__(self,
worker,
connector_name,
prefetch_count=None,
middlewares=None):
self.name = connector_name
self.worker = worker
self._consumers = {}
self._publishers = {}
self._endpoint_handlers = {}
self._default_handlers = {}
self._prefetch_count = prefetch_count
self._middlewares = MiddlewareStack(
middlewares if middlewares is not None else [])
def setup_middleware(self):
"""
Middleware setup happens here.
Subclasses should not override this unless they need to do nonstandard
middleware setup.
"""
middlewares = yield setup_middlewares_from_config(self, self.config)
self._middlewares = MiddlewareStack(middlewares)
def __init__(self, worker, connector_name, prefetch_count=None,
middlewares=None):
self.name = connector_name
self.worker = worker
self._consumers = {}
self._publishers = {}
self._endpoint_handlers = {}
self._default_handlers = {}
self._prefetch_count = prefetch_count
self._middlewares = MiddlewareStack(middlewares
if middlewares is not None else [])
def __init__(self, config):
self.transport_publisher = {}
self.transport_names = config.get('transport_names', [])
for transport in self.transport_names:
self.transport_publisher[transport] = self.DummyPublisher()
self.exposed_publisher = {}
self.exposed_event_publisher = {}
self.exposed_names = config.get('exposed_names', [])
for exposed in self.exposed_names:
self.exposed_publisher[exposed] = self.DummyPublisher()
self.exposed_event_publisher[exposed] = self.DummyPublisher()
self._middlewares = MiddlewareStack([])
class BaseConnector(object):
"""Base class for 'connector' objects.
A connector encapsulates the 'inbound', 'outbound' and 'event' publishers
and consumers required by vumi workers and avoids having to operate on them
individually all over the place.
"""
def __init__(self, worker, connector_name, prefetch_count=None,
middlewares=None):
self.name = connector_name
self.worker = worker
self._consumers = {}
self._publishers = {}
self._endpoint_handlers = {}
self._default_handlers = {}
self._prefetch_count = prefetch_count
self._middlewares = MiddlewareStack(middlewares
if middlewares is not None else [])
def _rkey(self, mtype):
return '%s.%s' % (self.name, mtype)
def setup(self):
raise NotImplementedError()
def teardown(self):
d = gatherResults([c.stop() for c in self._consumers.values()])
d.addCallback(lambda r: self._middlewares.teardown())
return d
@property
def paused(self):
return all(consumer.paused
for consumer in self._consumers.itervalues())
def pause(self):
return gatherResults([
consumer.pause() for consumer in self._consumers.itervalues()])
def unpause(self):
# This doesn't return a deferred.
for consumer in self._consumers.values():
consumer.unpause()
@inlineCallbacks
def _setup_publisher(self, mtype):
publisher = yield self.worker.publish_to(self._rkey(mtype))
self._publishers[mtype] = publisher
returnValue(publisher)
@inlineCallbacks
def _setup_consumer(self, mtype, msg_class, default_handler):
def handler(msg):
return self._consume_message(mtype, msg)
consumer = yield self.worker.consume(
self._rkey(mtype), handler, message_class=msg_class, paused=True,
prefetch_count=self._prefetch_count)
self._consumers[mtype] = consumer
self._set_default_endpoint_handler(mtype, default_handler)
returnValue(consumer)
def _set_endpoint_handler(self, mtype, handler, endpoint_name):
if endpoint_name is None:
endpoint_name = TransportMessage.DEFAULT_ENDPOINT_NAME
handlers = self._endpoint_handlers.setdefault(mtype, {})
handlers[endpoint_name] = handler
def _set_default_endpoint_handler(self, mtype, handler):
self._endpoint_handlers.setdefault(mtype, {})
self._default_handlers[mtype] = handler
def _consume_message(self, mtype, msg):
endpoint_name = msg.get_routing_endpoint()
handler = self._endpoint_handlers[mtype].get(endpoint_name)
if handler is None:
handler = self._default_handlers.get(mtype)
d = self._middlewares.apply_consume(mtype, msg, self.name)
d.addCallback(handler)
return d.addErrback(self._ignore_message, msg)
def _publish_message(self, mtype, msg, endpoint_name):
if endpoint_name is not None:
msg.set_routing_endpoint(endpoint_name)
d = self._middlewares.apply_publish(mtype, msg, self.name)
return d.addCallback(self._publishers[mtype].publish_message)
def _ignore_message(self, failure, msg):
failure.trap(IgnoreMessage)
log.debug("Ignoring msg due to %r: %r" % (failure.value, msg))
class Transport(Worker):
"""
Base class for transport workers.
The following attributes are available for subclasses to control behaviour:
* :attr:`start_message_consumer` -- Set to ``False`` if the message
consumer should not be started. The subclass is responsible for starting
it in this case.
"""
SUPPRESS_FAILURE_EXCEPTIONS = True
transport_name = None
start_message_consumer = True
@inlineCallbacks
def startWorker(self):
"""
Set up basic transport worker stuff.
You shouldn't have to override this in subclasses.
"""
self._consumers = []
self._validate_config()
if 'TRANSPORT_NAME' in self.config:
log.msg("NOTE: 'TRANSPORT_NAME' in config is deprecated. "
"Use 'transport_name' instead.")
self.config.setdefault('transport_name',
self.config['TRANSPORT_NAME'])
self.transport_name = self.config['transport_name']
self.concurrent_sends = self.config.get('concurrent_sends')
yield self._setup_failure_publisher()
yield self._setup_message_publisher()
yield self._setup_event_publisher()
yield self.setup_middleware()
yield self.setup_transport()
self.message_consumer = None
if self.start_message_consumer:
yield self._setup_message_consumer()
@inlineCallbacks
def stopWorker(self):
while self._consumers:
consumer = self._consumers.pop()
yield consumer.stop()
yield self.teardown_transport()
def get_rkey(self, mtype):
return '%s.%s' % (self.transport_name, mtype)
def publish_rkey(self, name):
return self.publish_to(self.get_rkey(name))
def _validate_config(self):
if 'transport_name' not in self.config:
raise ConfigError("Missing 'transport_name' field in config.")
return self.validate_config()
def validate_config(self):
"""
Transport-specific config validation happens in here.
Subclasses may override this method to perform extra config validation.
"""
pass
def setup_transport(self):
"""
All transport_specific setup should happen in here.
Subclasses should override this method to perform extra setup.
"""
pass
def teardown_transport(self):
"""
Clean-up of setup done in setup_transport should happen here.
"""
pass
@inlineCallbacks
def setup_middleware(self):
"""
Middleware setup happens here.
Subclasses should not override this unless they need to do nonstandard
middleware setup.
"""
middlewares = yield setup_middlewares_from_config(self, self.config)
self._middlewares = MiddlewareStack(middlewares)
@inlineCallbacks
def _setup_message_publisher(self):
self.message_publisher = yield self.publish_rkey('inbound')
@inlineCallbacks
def _setup_message_consumer(self):
if self.message_consumer is not None:
log.msg("Consumer already exists, not restarting.")
return
self.message_consumer = yield self.consume(
self.get_rkey('outbound'),
self._process_message,
message_class=TransportUserMessage)
self._consumers.append(self.message_consumer)
# Apply concurrency throttling if we need to.
if self.concurrent_sends is not None:
yield self.message_consumer.channel.basic_qos(
0, int(self.concurrent_sends), False)
def _teardown_message_consumer(self):
if self.message_consumer is None:
log.msg("Consumer does not exist, not stopping.")
return
if self.message_consumer in self._consumers:
self._consumers.remove(self.message_consumer)
consumer, self.message_consumer = self.message_consumer, None
return consumer.stop()
@inlineCallbacks
def _setup_event_publisher(self):
self.event_publisher = yield self.publish_rkey('event')
@inlineCallbacks
def _setup_failure_publisher(self):
self.failure_publisher = yield self.publish_rkey('failures')
def send_failure(self, message, exception, traceback):
"""Send a failure report."""
try:
failure_code = getattr(exception, "failure_code",
FailureMessage.FC_UNSPECIFIED)
failure_msg = FailureMessage(message=message.payload,
failure_code=failure_code,
reason=traceback)
d = self._middlewares.apply_publish("failure", failure_msg,
self.transport_name)
d.addCallback(self.failure_publisher.publish_message)
d.addCallback(lambda _f: self.failure_published())
except:
log.err("Error publishing failure: %s, %s, %s" %
(message, exception, traceback))
raise
return d
def failure_published(self):
pass
def publish_message(self, **kw):
"""
Publish a :class:`TransportUserMessage` message.
Some default parameters are handled, so subclasses don't have
to provide a lot of boilerplate.
"""
kw.setdefault('transport_name', self.transport_name)
kw.setdefault('transport_metadata', {})
msg = TransportUserMessage(**kw)
d = self._middlewares.apply_publish("inbound", msg,
self.transport_name)
d.addCallback(self.message_publisher.publish_message)
return d
def publish_event(self, **kw):
"""
Publish a :class:`TransportEvent` message.
Some default parameters are handled, so subclasses don't have
to provide a lot of boilerplate.
"""
kw.setdefault('transport_name', self.transport_name)
kw.setdefault('transport_metadata', {})
event = TransportEvent(**kw)
d = self._middlewares.apply_publish("event", event,
self.transport_name)
d.addCallback(self.event_publisher.publish_message)
return d
def publish_ack(self, user_message_id, sent_message_id, **kw):
"""
Helper method for publishing an ``ack`` event.
"""
return self.publish_event(user_message_id=user_message_id,
sent_message_id=sent_message_id,
event_type='ack',
**kw)
def publish_delivery_report(self, user_message_id, delivery_status, **kw):
"""
Helper method for publishing a ``delivery_report`` event.
"""
return self.publish_event(user_message_id=user_message_id,
delivery_status=delivery_status,
event_type='delivery_report',
**kw)
def _process_message(self, message):
def _send_failure(f):
self.send_failure(message, f.value, f.getTraceback())
log.err(f)
if self.SUPPRESS_FAILURE_EXCEPTIONS:
return None
return f
d = self._middlewares.apply_consume("outbound", message,
self.transport_name)
d.addCallback(self.handle_outbound_message)
d.addErrback(_send_failure)
return d
def handle_outbound_message(self, message):
"""
This must be overridden to read outbound messages and do the right
thing with them.
"""
raise NotImplementedError()
@staticmethod
def generate_message_id():
"""
Generate a message id.
"""
return TransportUserMessage.generate_id()
class ApplicationWorker(Worker):
"""Base class for an application worker.
Handles :class:`vumi.message.TransportUserMessage` and
:class:`vumi.message.TransportEvent` messages.
Application workers may send outgoing messages using
:meth:`reply_to` (for replies to incoming messages) or
:meth:`send_to` (for messages that are not replies).
Messages sent via :meth:`send_to` pass optional additional data
from configuration to the TransportUserMessage constructor, based
on the tag parameter passed to send_to. This usually contains
information useful for routing the message.
An example :meth:`send_to` configuration might look like::
- send_to:
- default:
transport_name: sms_transport
Currently 'transport_name' **must** be defined for each send_to
section since all existing dispatchers rely on this for routing
outbound messages.
The available tags are defined by the :attr:`SEND_TO_TAGS` class
attribute. Sub-classes must override this attribute with a set of
tag names if they wish to use :meth:`send_to`. If applications
have only a single tag, it is suggested to name that tag `default`
(this makes calling `send_to` easier since the value of the tag
parameter may be omitted).
By default :attr:`SEND_TO_TAGS` is empty and all calls to
:meth:`send_to` will fail (this is to make it easy to identify
which tags an application requires `send_to` configuration for).
"""
transport_name = None
start_message_consumer = True
SEND_TO_TAGS = frozenset([])
@inlineCallbacks
def startWorker(self):
log.msg('Starting a %s worker with config: %s'
% (self.__class__.__name__, self.config))
self._consumers = []
self._validate_config()
self.transport_name = self.config['transport_name']
self.send_to_options = self.config.get('send_to', {})
self._event_handlers = {
'ack': self.consume_ack,
'delivery_report': self.consume_delivery_report,
}
self._session_handlers = {
SESSION_NEW: self.new_session,
SESSION_CLOSE: self.close_session,
}
yield self._setup_transport_publisher()
yield self.setup_middleware()
yield self.setup_application()
if self.start_message_consumer:
yield self._setup_transport_consumer()
yield self._setup_event_consumer()
@inlineCallbacks
def stopWorker(self):
while self._consumers:
consumer = self._consumers.pop()
yield consumer.stop()
yield self.teardown_application()
def _validate_config(self):
if 'transport_name' not in self.config:
raise ConfigError("Missing 'transport_name' field in config.")
send_to_options = self.config.get('send_to', {})
for tag in self.SEND_TO_TAGS:
if tag not in send_to_options:
raise ConfigError("No configuration for send_to tag %r but"
" at least a transport_name is required."
% (tag,))
if 'transport_name' not in send_to_options[tag]:
raise ConfigError("The configuration for send_to tag %r must"
" contain a transport_name." % (tag,))
return self.validate_config()
def validate_config(self):
"""
Application-specific config validation happens in here.
Subclasses may override this method to perform extra config validation.
"""
pass
def setup_application(self):
"""
All application specific setup should happen in here.
Subclasses should override this method to perform extra setup.
"""
pass
def teardown_application(self):
"""
Clean-up of setup done in setup_application should happen here.
"""
pass
@inlineCallbacks
def setup_middleware(self):
"""
Middleware setup happens here.
Subclasses should not override this unless they need to do nonstandard
middleware setup.
"""
middlewares = yield setup_middlewares_from_config(self, self.config)
self._middlewares = MiddlewareStack(middlewares)
def _dispatch_event_raw(self, event):
event_type = event.get('event_type')
handler = self._event_handlers.get(event_type,
self.consume_unknown_event)
return handler(event)
def dispatch_event(self, event):
"""Dispatch to event_type specific handlers."""
d = self._middlewares.apply_consume("event", event,
self.transport_name)
d.addCallback(self._dispatch_event_raw)
return d
def consume_unknown_event(self, event):
log.msg("Unknown event type in message %r" % (event,))
def consume_ack(self, event):
"""Handle an ack message."""
pass
def consume_delivery_report(self, event):
"""Handle a delivery report."""
pass
def _dispatch_user_message_raw(self, message):
session_event = message.get('session_event')
handler = self._session_handlers.get(session_event,
self.consume_user_message)
return handler(message)
def dispatch_user_message(self, message):
"""Dispatch user messages to handler."""
d = self._middlewares.apply_consume("inbound", message,
self.transport_name)
d.addCallback(self._dispatch_user_message_raw)
return d
def consume_user_message(self, message):
"""Respond to user message."""
pass
def new_session(self, message):
"""Respond to a new session.
Defaults to calling consume_user_message.
"""
return self.consume_user_message(message)
def close_session(self, message):
"""Close a session.
The .reply_to() method should not be called when the session is closed.
"""
pass
def _publish_message(self, message):
d = self._middlewares.apply_publish("outbound", message,
self.transport_name)
d.addCallback(self.transport_publisher.publish_message)
return d
def reply_to(self, original_message, content, continue_session=True,
**kws):
reply = original_message.reply(content, continue_session, **kws)
return self._publish_message(reply)
def reply_to_group(self, original_message, content, continue_session=True,
**kws):
reply = original_message.reply_group(content, continue_session, **kws)
return self._publish_message(reply)
def send_to(self, to_addr, content, tag='default', **kw):
if tag not in self.SEND_TO_TAGS:
raise ValueError("Tag %r not defined in SEND_TO_TAGS" % (tag,))
options = copy.deepcopy(self.send_to_options[tag])
options.update(kw)
msg = TransportUserMessage.send(to_addr, content, **options)
return self._publish_message(msg)
@inlineCallbacks
def _setup_transport_publisher(self):
self.transport_publisher = yield self.publish_to(
'%(transport_name)s.outbound' % self.config)
@inlineCallbacks
def _setup_transport_consumer(self):
self.transport_consumer = yield self.consume(
'%(transport_name)s.inbound' % self.config,
self.dispatch_user_message,
message_class=TransportUserMessage)
self._consumers.append(self.transport_consumer)
@inlineCallbacks
def _setup_event_consumer(self):
self.transport_event_consumer = yield self.consume(
'%(transport_name)s.event' % self.config,
self.dispatch_event,
message_class=TransportEvent)
self._consumers.append(self.transport_event_consumer)
class ApplicationWorker(Worker):
"""Base class for an application worker.
Handles :class:`vumi.message.TransportUserMessage` and
:class:`vumi.message.TransportEvent` messages.
Application workers may send outgoing messages using
:meth:`reply_to` (for replies to incoming messages) or
:meth:`send_to` (for messages that are not replies).
Messages sent via :meth:`send_to` pass optional additional data
from configuration to the TransportUserMessage constructor, based
on the tag parameter passed to send_to. This usually contains
information useful for routing the message.
An example :meth:`send_to` configuration might look like::
- send_to:
- default:
transport_name: sms_transport
Currently 'transport_name' **must** be defined for each send_to
section since all existing dispatchers rely on this for routing
outbound messages.
The available tags are defined by the :attr:`SEND_TO_TAGS` class
attribute. Sub-classes must override this attribute with a set of
tag names if they wish to use :meth:`send_to`. If applications
have only a single tag, it is suggested to name that tag `default`
(this makes calling `send_to` easier since the value of the tag
parameter may be omitted).
By default :attr:`SEND_TO_TAGS` is empty and all calls to
:meth:`send_to` will fail (this is to make it easy to identify
which tags an application requires `send_to` configuration for).
"""
transport_name = None
start_message_consumer = True
SEND_TO_TAGS = frozenset([])
@inlineCallbacks
def startWorker(self):
log.msg('Starting a %s worker with config: %s' %
(self.__class__.__name__, self.config))
self._consumers = []
self._validate_config()
self.transport_name = self.config['transport_name']
self.send_to_options = self.config.get('send_to', {})
self._event_handlers = {
'ack': self.consume_ack,
'delivery_report': self.consume_delivery_report,
}
self._session_handlers = {
SESSION_NEW: self.new_session,
SESSION_CLOSE: self.close_session,
}
yield self._setup_transport_publisher()
yield self.setup_middleware()
yield self.setup_application()
if self.start_message_consumer:
yield self._setup_transport_consumer()
yield self._setup_event_consumer()
@inlineCallbacks
def stopWorker(self):
while self._consumers:
consumer = self._consumers.pop()
yield consumer.stop()
yield self.teardown_application()
def _validate_config(self):
if 'transport_name' not in self.config:
raise ConfigError("Missing 'transport_name' field in config.")
send_to_options = self.config.get('send_to', {})
for tag in self.SEND_TO_TAGS:
if tag not in send_to_options:
raise ConfigError("No configuration for send_to tag %r but"
" at least a transport_name is required." %
(tag, ))
if 'transport_name' not in send_to_options[tag]:
raise ConfigError("The configuration for send_to tag %r must"
" contain a transport_name." % (tag, ))
return self.validate_config()
def validate_config(self):
"""
Application-specific config validation happens in here.
Subclasses may override this method to perform extra config validation.
"""
pass
def setup_application(self):
"""
All application specific setup should happen in here.
Subclasses should override this method to perform extra setup.
"""
pass
def teardown_application(self):
"""
Clean-up of setup done in setup_application should happen here.
"""
pass
@inlineCallbacks
def setup_middleware(self):
"""
Middleware setup happens here.
Subclasses should not override this unless they need to do nonstandard
middleware setup.
"""
middlewares = yield setup_middlewares_from_config(self, self.config)
self._middlewares = MiddlewareStack(middlewares)
def _dispatch_event_raw(self, event):
event_type = event.get('event_type')
handler = self._event_handlers.get(event_type,
self.consume_unknown_event)
return handler(event)
def dispatch_event(self, event):
"""Dispatch to event_type specific handlers."""
d = self._middlewares.apply_consume("event", event,
self.transport_name)
d.addCallback(self._dispatch_event_raw)
return d
def consume_unknown_event(self, event):
log.msg("Unknown event type in message %r" % (event, ))
def consume_ack(self, event):
"""Handle an ack message."""
pass
def consume_delivery_report(self, event):
"""Handle a delivery report."""
pass
def _dispatch_user_message_raw(self, message):
session_event = message.get('session_event')
handler = self._session_handlers.get(session_event,
self.consume_user_message)
return handler(message)
def dispatch_user_message(self, message):
"""Dispatch user messages to handler."""
d = self._middlewares.apply_consume("inbound", message,
self.transport_name)
d.addCallback(self._dispatch_user_message_raw)
return d
def consume_user_message(self, message):
"""Respond to user message."""
pass
def new_session(self, message):
"""Respond to a new session.
Defaults to calling consume_user_message.
"""
return self.consume_user_message(message)
def close_session(self, message):
"""Close a session.
The .reply_to() method should not be called when the session is closed.
"""
pass
def _publish_message(self, message):
d = self._middlewares.apply_publish("outbound", message,
self.transport_name)
d.addCallback(self.transport_publisher.publish_message)
return d
def reply_to(self,
original_message,
content,
continue_session=True,
**kws):
reply = original_message.reply(content, continue_session, **kws)
return self._publish_message(reply)
def reply_to_group(self,
original_message,
content,
continue_session=True,
**kws):
reply = original_message.reply_group(content, continue_session, **kws)
return self._publish_message(reply)
def send_to(self, to_addr, content, tag='default', **kw):
if tag not in self.SEND_TO_TAGS:
raise ValueError("Tag %r not defined in SEND_TO_TAGS" % (tag, ))
options = copy.deepcopy(self.send_to_options[tag])
options.update(kw)
msg = TransportUserMessage.send(to_addr, content, **options)
return self._publish_message(msg)
@inlineCallbacks
def _setup_transport_publisher(self):
self.transport_publisher = yield self.publish_to(
'%(transport_name)s.outbound' % self.config)
@inlineCallbacks
def _setup_transport_consumer(self):
self.transport_consumer = yield self.consume(
'%(transport_name)s.inbound' % self.config,
self.dispatch_user_message,
message_class=TransportUserMessage)
self._consumers.append(self.transport_consumer)
@inlineCallbacks
def _setup_event_consumer(self):
self.transport_event_consumer = yield self.consume(
'%(transport_name)s.event' % self.config,
self.dispatch_event,
message_class=TransportEvent)
self._consumers.append(self.transport_event_consumer)
def setup_middleware(self):
middlewares = yield setup_middlewares_from_config(self, self.config)
self._middlewares = MiddlewareStack(middlewares)
class BaseDispatchWorker(Worker):
"""Base class for a dispatch worker.
"""
@inlineCallbacks
def startWorker(self):
log.msg('Starting a %s dispatcher with config: %s' %
(self.__class__.__name__, self.config))
self.amqp_prefetch_count = self.config.get('amqp_prefetch_count', 20)
yield self.setup_endpoints()
yield self.setup_middleware()
yield self.setup_router()
yield self.setup_transport_publishers()
yield self.setup_exposed_publishers()
yield self.setup_transport_consumers()
yield self.setup_exposed_consumers()
consumers = (self.exposed_consumer.values() +
self.transport_consumer.values() +
self.transport_event_consumer.values())
for consumer in consumers:
consumer.unpause()
@inlineCallbacks
def stopWorker(self):
yield self.teardown_router()
yield self.teardown_middleware()
def setup_endpoints(self):
self.transport_names = self.config.get('transport_names', [])
self.exposed_names = self.config.get('exposed_names', [])
@inlineCallbacks
def setup_middleware(self):
middlewares = yield setup_middlewares_from_config(self, self.config)
self._middlewares = MiddlewareStack(middlewares)
def teardown_middleware(self):
return self._middlewares.teardown()
def setup_router(self):
router_cls = load_class_by_string(self.config['router_class'])
self._router = router_cls(self, self.config)
return maybeDeferred(self._router.setup_routing)
def teardown_router(self):
return maybeDeferred(self._router.teardown_routing)
@inlineCallbacks
def setup_transport_publishers(self):
self.transport_publisher = {}
for transport_name in self.transport_names:
self.transport_publisher[transport_name] = yield self.publish_to(
'%s.outbound' % (transport_name, ))
@inlineCallbacks
def setup_transport_consumers(self):
self.transport_consumer = {}
self.transport_event_consumer = {}
for transport_name in self.transport_names:
self.transport_consumer[transport_name] = yield self.consume(
'%s.inbound' % (transport_name, ),
functools.partial(self.dispatch_inbound_message,
transport_name),
message_class=TransportUserMessage,
paused=True,
prefetch_count=self.amqp_prefetch_count)
for transport_name in self.transport_names:
self.transport_event_consumer[transport_name] = yield self.consume(
'%s.event' % (transport_name, ),
functools.partial(self.dispatch_inbound_event, transport_name),
message_class=TransportEvent,
paused=True,
prefetch_count=self.amqp_prefetch_count)
@inlineCallbacks
def setup_exposed_publishers(self):
self.exposed_publisher = {}
self.exposed_event_publisher = {}
for exposed_name in self.exposed_names:
self.exposed_publisher[exposed_name] = yield self.publish_to(
'%s.inbound' % (exposed_name, ))
for exposed_name in self.exposed_names:
self.exposed_event_publisher[exposed_name] = yield self.publish_to(
'%s.event' % (exposed_name, ))
@inlineCallbacks
def setup_exposed_consumers(self):
self.exposed_consumer = {}
for exposed_name in self.exposed_names:
self.exposed_consumer[exposed_name] = yield self.consume(
'%s.outbound' % (exposed_name, ),
functools.partial(self.dispatch_outbound_message,
exposed_name),
message_class=TransportUserMessage,
paused=True,
prefetch_count=self.amqp_prefetch_count)
def dispatch_inbound_message(self, endpoint, msg):
d = self._middlewares.apply_consume("inbound", msg, endpoint)
d.addCallback(self._router.dispatch_inbound_message)
return d
def dispatch_inbound_event(self, endpoint, msg):
d = self._middlewares.apply_consume("event", msg, endpoint)
d.addCallback(self._router.dispatch_inbound_event)
return d
def dispatch_outbound_message(self, endpoint, msg):
d = self._middlewares.apply_consume("outbound", msg, endpoint)
d.addCallback(self._router.dispatch_outbound_message)
return d
def publish_inbound_message(self, endpoint, msg):
d = self._middlewares.apply_publish("inbound", msg, endpoint)
d.addCallback(self.exposed_publisher[endpoint].publish_message)
return d
def publish_inbound_event(self, endpoint, msg):
d = self._middlewares.apply_publish("event", msg, endpoint)
d.addCallback(self.exposed_event_publisher[endpoint].publish_message)
return d
def publish_outbound_message(self, endpoint, msg):
d = self._middlewares.apply_publish("outbound", msg, endpoint)
d.addCallback(self.transport_publisher[endpoint].publish_message)
return d
def setup_middleware(self):
middlewares = yield setup_middlewares_from_config(self, self.config)
self._middlewares = MiddlewareStack(middlewares)
class BaseDispatchWorker(Worker):
"""Base class for a dispatch worker.
"""
@inlineCallbacks
def startWorker(self):
log.msg('Starting a %s dispatcher with config: %s'
% (self.__class__.__name__, self.config))
self.amqp_prefetch_count = self.config.get('amqp_prefetch_count', 20)
yield self.setup_endpoints()
yield self.setup_middleware()
yield self.setup_router()
yield self.setup_transport_publishers()
yield self.setup_exposed_publishers()
yield self.setup_transport_consumers()
yield self.setup_exposed_consumers()
consumers = (self.exposed_consumer.values() +
self.transport_consumer.values() +
self.transport_event_consumer.values())
for consumer in consumers:
consumer.unpause()
@inlineCallbacks
def stopWorker(self):
yield self.teardown_router()
yield self.teardown_middleware()
def setup_endpoints(self):
self.transport_names = self.config.get('transport_names', [])
self.exposed_names = self.config.get('exposed_names', [])
@inlineCallbacks
def setup_middleware(self):
middlewares = yield setup_middlewares_from_config(self, self.config)
self._middlewares = MiddlewareStack(middlewares)
def teardown_middleware(self):
return self._middlewares.teardown()
def setup_router(self):
router_cls = load_class_by_string(self.config['router_class'])
self._router = router_cls(self, self.config)
return maybeDeferred(self._router.setup_routing)
def teardown_router(self):
return maybeDeferred(self._router.teardown_routing)
@inlineCallbacks
def setup_transport_publishers(self):
self.transport_publisher = {}
for transport_name in self.transport_names:
self.transport_publisher[transport_name] = yield self.publish_to(
'%s.outbound' % (transport_name,))
@inlineCallbacks
def setup_transport_consumers(self):
self.transport_consumer = {}
self.transport_event_consumer = {}
for transport_name in self.transport_names:
self.transport_consumer[transport_name] = yield self.consume(
'%s.inbound' % (transport_name,),
functools.partial(self.dispatch_inbound_message,
transport_name),
message_class=TransportUserMessage, paused=True,
prefetch_count=self.amqp_prefetch_count)
for transport_name in self.transport_names:
self.transport_event_consumer[transport_name] = yield self.consume(
'%s.event' % (transport_name,),
functools.partial(self.dispatch_inbound_event, transport_name),
message_class=TransportEvent, paused=True,
prefetch_count=self.amqp_prefetch_count)
@inlineCallbacks
def setup_exposed_publishers(self):
self.exposed_publisher = {}
self.exposed_event_publisher = {}
for exposed_name in self.exposed_names:
self.exposed_publisher[exposed_name] = yield self.publish_to(
'%s.inbound' % (exposed_name,))
for exposed_name in self.exposed_names:
self.exposed_event_publisher[exposed_name] = yield self.publish_to(
'%s.event' % (exposed_name,))
@inlineCallbacks
def setup_exposed_consumers(self):
self.exposed_consumer = {}
for exposed_name in self.exposed_names:
self.exposed_consumer[exposed_name] = yield self.consume(
'%s.outbound' % (exposed_name,),
functools.partial(self.dispatch_outbound_message,
exposed_name),
message_class=TransportUserMessage, paused=True,
prefetch_count=self.amqp_prefetch_count)
def dispatch_inbound_message(self, endpoint, msg):
d = self._middlewares.apply_consume("inbound", msg, endpoint)
d.addCallback(self._router.dispatch_inbound_message)
return d
def dispatch_inbound_event(self, endpoint, msg):
d = self._middlewares.apply_consume("event", msg, endpoint)
d.addCallback(self._router.dispatch_inbound_event)
return d
def dispatch_outbound_message(self, endpoint, msg):
d = self._middlewares.apply_consume("outbound", msg, endpoint)
d.addCallback(self._router.dispatch_outbound_message)
return d
def publish_inbound_message(self, endpoint, msg):
d = self._middlewares.apply_publish("inbound", msg, endpoint)
d.addCallback(self.exposed_publisher[endpoint].publish_message)
return d
def publish_inbound_event(self, endpoint, msg):
d = self._middlewares.apply_publish("event", msg, endpoint)
d.addCallback(self.exposed_event_publisher[endpoint].publish_message)
return d
def publish_outbound_message(self, endpoint, msg):
d = self._middlewares.apply_publish("outbound", msg, endpoint)
d.addCallback(self.transport_publisher[endpoint].publish_message)
return d
class BaseConnector(object):
"""Base class for 'connector' objects.
A connector encapsulates the 'inbound', 'outbound' and 'event' publishers
and consumers required by vumi workers and avoids having to operate on them
individually all over the place.
"""
def __init__(self,
worker,
connector_name,
prefetch_count=None,
middlewares=None):
self.name = connector_name
self.worker = worker
self._consumers = {}
self._publishers = {}
self._endpoint_handlers = {}
self._default_handlers = {}
self._prefetch_count = prefetch_count
self._middlewares = MiddlewareStack(
middlewares if middlewares is not None else [])
def _rkey(self, mtype):
return '%s.%s' % (self.name, mtype)
def setup(self):
raise NotImplementedError()
def teardown(self):
d = gatherResults([c.stop() for c in self._consumers.values()])
d.addCallback(lambda r: self._middlewares.teardown())
return d
@property
def paused(self):
return all(consumer.paused
for consumer in self._consumers.itervalues())
def pause(self):
return gatherResults(
[consumer.pause() for consumer in self._consumers.itervalues()])
def unpause(self):
# This doesn't return a deferred.
for consumer in self._consumers.values():
consumer.unpause()
@inlineCallbacks
def _setup_publisher(self, mtype):
publisher = yield self.worker.publish_to(self._rkey(mtype))
self._publishers[mtype] = publisher
returnValue(publisher)
@inlineCallbacks
def _setup_consumer(self, mtype, msg_class, default_handler):
def handler(msg):
return self._consume_message(mtype, msg)
consumer = yield self.worker.consume(
self._rkey(mtype),
handler,
message_class=msg_class,
paused=True,
prefetch_count=self._prefetch_count)
self._consumers[mtype] = consumer
self._set_default_endpoint_handler(mtype, default_handler)
returnValue(consumer)
def _set_endpoint_handler(self, mtype, handler, endpoint_name):
if endpoint_name is None:
endpoint_name = TransportMessage.DEFAULT_ENDPOINT_NAME
handlers = self._endpoint_handlers.setdefault(mtype, {})
handlers[endpoint_name] = handler
def _set_default_endpoint_handler(self, mtype, handler):
self._endpoint_handlers.setdefault(mtype, {})
self._default_handlers[mtype] = handler
def _consume_message(self, mtype, msg):
endpoint_name = msg.get_routing_endpoint()
handler = self._endpoint_handlers[mtype].get(endpoint_name)
if handler is None:
handler = self._default_handlers.get(mtype)
d = self._middlewares.apply_consume(mtype, msg, self.name)
d.addCallback(handler)
return d.addErrback(self._ignore_message, msg)
def _publish_message(self, mtype, msg, endpoint_name):
if endpoint_name is not None:
msg.set_routing_endpoint(endpoint_name)
d = self._middlewares.apply_publish(mtype, msg, self.name)
return d.addCallback(self._publishers[mtype].publish_message)
def _ignore_message(self, failure, msg):
failure.trap(IgnoreMessage)
log.debug("Ignoring msg due to %r: %r" % (failure.value, msg))
class Transport(Worker):
"""
Base class for transport workers.
The following attributes are available for subclasses to control behaviour:
* :attr:`start_message_consumer` -- Set to ``False`` if the message
consumer should not be started. The subclass is responsible for starting
it in this case.
"""
SUPPRESS_FAILURE_EXCEPTIONS = True
transport_name = None
start_message_consumer = True
@inlineCallbacks
def startWorker(self):
"""
Set up basic transport worker stuff.
You shouldn't have to override this in subclasses.
"""
self._consumers = []
self._validate_config()
if 'TRANSPORT_NAME' in self.config:
log.msg("NOTE: 'TRANSPORT_NAME' in config is deprecated. "
"Use 'transport_name' instead.")
self.config.setdefault('transport_name',
self.config['TRANSPORT_NAME'])
self.transport_name = self.config['transport_name']
self.concurrent_sends = self.config.get('concurrent_sends')
yield self._setup_failure_publisher()
yield self._setup_message_publisher()
yield self._setup_event_publisher()
yield self.setup_middleware()
yield self.setup_transport()
self.message_consumer = None
if self.start_message_consumer:
yield self._setup_message_consumer()
@inlineCallbacks
def stopWorker(self):
while self._consumers:
consumer = self._consumers.pop()
yield consumer.stop()
yield self.teardown_transport()
def get_rkey(self, mtype):
return '%s.%s' % (self.transport_name, mtype)
def publish_rkey(self, name):
return self.publish_to(self.get_rkey(name))
def _validate_config(self):
if 'transport_name' not in self.config:
raise ConfigError("Missing 'transport_name' field in config.")
return self.validate_config()
def validate_config(self):
"""
Transport-specific config validation happens in here.
Subclasses may override this method to perform extra config validation.
"""
pass
def setup_transport(self):
"""
All transport_specific setup should happen in here.
Subclasses should override this method to perform extra setup.
"""
pass
def teardown_transport(self):
"""
Clean-up of setup done in setup_transport should happen here.
"""
pass
@inlineCallbacks
def setup_middleware(self):
"""
Middleware setup happens here.
Subclasses should not override this unless they need to do nonstandard
middleware setup.
"""
middlewares = yield setup_middlewares_from_config(self, self.config)
self._middlewares = MiddlewareStack(middlewares)
@inlineCallbacks
def _setup_message_publisher(self):
self.message_publisher = yield self.publish_rkey('inbound')
@inlineCallbacks
def _setup_message_consumer(self):
if self.message_consumer is not None:
log.msg("Consumer already exists, not restarting.")
return
self.message_consumer = yield self.consume(
self.get_rkey('outbound'), self._process_message,
message_class=TransportUserMessage)
self._consumers.append(self.message_consumer)
# Apply concurrency throttling if we need to.
if self.concurrent_sends is not None:
yield self.message_consumer.channel.basic_qos(
0, int(self.concurrent_sends), False)
def _teardown_message_consumer(self):
if self.message_consumer is None:
log.msg("Consumer does not exist, not stopping.")
return
if self.message_consumer in self._consumers:
self._consumers.remove(self.message_consumer)
consumer, self.message_consumer = self.message_consumer, None
return consumer.stop()
@inlineCallbacks
def _setup_event_publisher(self):
self.event_publisher = yield self.publish_rkey('event')
@inlineCallbacks
def _setup_failure_publisher(self):
self.failure_publisher = yield self.publish_rkey('failures')
def send_failure(self, message, exception, traceback):
"""Send a failure report."""
try:
failure_code = getattr(exception, "failure_code",
FailureMessage.FC_UNSPECIFIED)
failure_msg = FailureMessage(
message=message.payload, failure_code=failure_code,
reason=traceback)
d = self._middlewares.apply_publish("failure", failure_msg,
self.transport_name)
d.addCallback(self.failure_publisher.publish_message)
d.addCallback(lambda _f: self.failure_published())
except:
log.err("Error publishing failure: %s, %s, %s"
% (message, exception, traceback))
raise
return d
def failure_published(self):
pass
def publish_message(self, **kw):
"""
Publish a :class:`TransportUserMessage` message.
Some default parameters are handled, so subclasses don't have
to provide a lot of boilerplate.
"""
kw.setdefault('transport_name', self.transport_name)
kw.setdefault('transport_metadata', {})
msg = TransportUserMessage(**kw)
d = self._middlewares.apply_publish("inbound", msg,
self.transport_name)
d.addCallback(self.message_publisher.publish_message)
return d
def publish_event(self, **kw):
"""
Publish a :class:`TransportEvent` message.
Some default parameters are handled, so subclasses don't have
to provide a lot of boilerplate.
"""
kw.setdefault('transport_name', self.transport_name)
kw.setdefault('transport_metadata', {})
event = TransportEvent(**kw)
d = self._middlewares.apply_publish("event", event,
self.transport_name)
d.addCallback(self.event_publisher.publish_message)
return d
def publish_ack(self, user_message_id, sent_message_id, **kw):
"""
Helper method for publishing an ``ack`` event.
"""
return self.publish_event(user_message_id=user_message_id,
sent_message_id=sent_message_id,
event_type='ack', **kw)
def publish_delivery_report(self, user_message_id, delivery_status, **kw):
"""
Helper method for publishing a ``delivery_report`` event.
"""
return self.publish_event(user_message_id=user_message_id,
delivery_status=delivery_status,
event_type='delivery_report', **kw)
def _process_message(self, message):
def _send_failure(f):
self.send_failure(message, f.value, f.getTraceback())
log.err(f)
if self.SUPPRESS_FAILURE_EXCEPTIONS:
return None
return f
d = self._middlewares.apply_consume("outbound", message,
self.transport_name)
d.addCallback(self.handle_outbound_message)
d.addErrback(_send_failure)
return d
def handle_outbound_message(self, message):
"""
This must be overridden to read outbound messages and do the right
thing with them.
"""
raise NotImplementedError()
@staticmethod
def generate_message_id():
"""
Generate a message id.
"""
return TransportUserMessage.generate_id()