def setup_application(self):
self.redis = yield TxRedisManager.from_config(
self.config['redis_manager'])
self.inbounds = InboundMessageStore(self.redis,
self.config['inbound_ttl'])
self.outbounds = OutboundMessageStore(self.redis,
self.config['outbound_ttl'])
self.message_rate = MessageRateStore(self.redis)
if self.config.get('message_queue') is not None:
self.ro_connector = yield self.setup_ro_connector(
self.config['message_queue'])
self.ro_connector.set_outbound_handler(self._publish_message)
def setup(self, redis=None, message_sender=None):
if redis is None:
redis = yield TxRedisManager.from_config(self.redis_config)
if message_sender is None:
message_sender = MessageSender('amqp-spec-0-8.xml',
self.amqp_config)
self.redis = redis
self.message_sender = message_sender
self.message_sender.setServiceParent(self.service)
self.inbounds = InboundMessageStore(self.redis,
self.config.inbound_message_ttl)
self.outbounds = OutboundMessageStore(self.redis,
self.config.outbound_message_ttl)
self.message_rate = MessageRateStore(self.redis)
self.plugins = []
for plugin_config in self.config.plugins:
cls = load_class_by_string(plugin_config['type'])
plugin = cls()
yield plugin.start_plugin(plugin_config, self.config)
self.plugins.append(plugin)
yield Channel.start_all_channels(self.redis, self.config, self.service,
self.plugins)
def setup_application(self):
self.redis = yield TxRedisManager.from_config(
self.config['redis_manager'])
self.inbounds = InboundMessageStore(
self.redis, self.config['inbound_ttl'])
self.outbounds = OutboundMessageStore(
self.redis, self.config['outbound_ttl'])
def setup_application(self):
self.redis = yield TxRedisManager.from_config(
self.config['redis_manager'])
self.inbounds = InboundMessageStore(
self.redis, self.config['inbound_ttl'])
self.outbounds = OutboundMessageStore(
self.redis, self.config['outbound_ttl'])
self.message_rate = MessageRateStore(self.redis)
if self.config.get('message_queue') is not None:
self.ro_connector = yield self.setup_ro_connector(
self.config['message_queue'])
self.ro_connector.set_outbound_handler(
self._publish_message)
class MessageForwardingWorker(ApplicationWorker):
'''This application worker consumes vumi messages placed on a configured
amqp queue, and sends them as HTTP requests with a JSON body to a
configured URL'''
CONFIG_CLASS = MessageForwardingConfig
@inlineCallbacks
def setup_application(self):
self.redis = yield TxRedisManager.from_config(
self.config['redis_manager'])
self.inbounds = InboundMessageStore(self.redis,
self.config['inbound_ttl'])
self.outbounds = OutboundMessageStore(self.redis,
self.config['outbound_ttl'])
self.message_rate = MessageRateStore(self.redis)
if self.config.get('message_queue') is not None:
self.ro_connector = yield self.setup_ro_connector(
self.config['message_queue'])
self.ro_connector.set_outbound_handler(self._publish_message)
@inlineCallbacks
def teardown_application(self):
yield self.redis.close_manager()
@property
def channel_id(self):
return self.config['transport_name']
@inlineCallbacks
def consume_user_message(self, message):
'''Sends the vumi message as an HTTP request to the configured URL'''
yield self.inbounds.store_vumi_message(self.channel_id, message)
msg = api_from_message(message)
if self.config.get('mo_message_url') is not None:
resp = yield post(self.config['mo_message_url'], msg)
if request_failed(resp):
logging.exception(
'Error sending message, received HTTP code %r with body %r'
'. Message: %r' % (resp.code, (yield resp.content()), msg))
if self.config.get('message_queue') is not None:
yield self.ro_connector.publish_inbound(message)
yield self._increment_metric('inbound')
@inlineCallbacks
def store_and_forward_event(self, event):
'''Store the event in the message store, POST it to the correct
URL.'''
yield self._store_event(event)
yield self._forward_event(event)
yield self._count_event(event)
def _increment_metric(self, label):
return self.message_rate.increment(self.channel_id, label,
self.config['metric_window'])
def _count_event(self, event):
if event['event_type'] == 'ack':
return self._increment_metric('submitted')
if event['event_type'] == 'nack':
return self._increment_metric('rejected')
if event['event_type'] == 'delivery_report':
if event['delivery_status'] == 'pending':
return self._increment_metric('delivery_pending')
if event['delivery_status'] == 'failed':
return self._increment_metric('delivery_failed')
if event['delivery_status'] == 'delivered':
return self._increment_metric('delivery_succeeded')
def _store_event(self, event):
'''Stores the event in the message store'''
message_id = event['user_message_id']
return self.outbounds.store_event(self.channel_id, message_id, event)
@inlineCallbacks
def _forward_event(self, event):
'''Forward the event to the correct places.'''
yield self._forward_event_http(event)
yield self._forward_event_amqp(event)
@inlineCallbacks
def _forward_event_http(self, event):
'''POST the event to the correct URL'''
url = yield self._get_event_url(event)
if url is None:
return
msg = api_from_event(self.channel_id, event)
if msg['event_type'] is None:
logging.exception("Discarding unrecognised event %r" % (event, ))
return
resp = yield post(url, msg)
if request_failed(resp):
logging.exception(
'Error sending event, received HTTP code %r with body %r. '
'Event: %r' % (resp.code, (yield resp.content()), event))
def _forward_event_amqp(self, event):
'''Put the event on the correct queue.'''
if self.config.get('message_queue') is not None:
return self.ro_connector.publish_event(event)
def consume_ack(self, event):
return self.store_and_forward_event(event)
def consume_nack(self, event):
return self.store_and_forward_event(event)
def consume_delivery_report(self, event):
return self.store_and_forward_event(event)
def _get_event_url(self, event):
msg_id = event['user_message_id']
return self.outbounds.load_event_url(self.channel_id, msg_id)
class MessageForwardingWorker(ApplicationWorker):
'''This application worker consumes vumi messages placed on a configured
amqp queue, and sends them as HTTP requests with a JSON body to a
configured URL'''
CONFIG_CLASS = MessageForwardingConfig
@inlineCallbacks
def setup_application(self):
self.redis = yield TxRedisManager.from_config(
self.config['redis_manager'])
self.inbounds = InboundMessageStore(
self.redis, self.config['inbound_ttl'])
self.outbounds = OutboundMessageStore(
self.redis, self.config['outbound_ttl'])
self.message_rate = MessageRateStore(self.redis)
if self.config.get('message_queue') is not None:
self.ro_connector = yield self.setup_ro_connector(
self.config['message_queue'])
self.ro_connector.set_outbound_handler(
self._publish_message)
@inlineCallbacks
def teardown_application(self):
if getattr(self, 'redis', None) is not None:
yield self.redis.close_manager()
@property
def channel_id(self):
return self.config['transport_name']
@inlineCallbacks
def consume_user_message(self, message):
'''Sends the vumi message as an HTTP request to the configured URL'''
yield self.inbounds.store_vumi_message(self.channel_id, message)
msg = api_from_message(message)
if self.config.get('mo_message_url') is not None:
config = self.get_static_config()
(url, auth) = self._split_url_and_credentials(
config.mo_message_url)
# Construct token auth headers if configured
auth_token = config.mo_message_url_auth_token
if auth_token:
headers = {
'Authorization': [
'Token %s' % (auth_token,)]
}
else:
headers = {}
resp = yield post(url, msg,
timeout=config.mo_message_url_timeout,
auth=auth, headers=headers)
if resp and request_failed(resp):
logging.exception(
'Error sending message, received HTTP code %r with body %r'
'. Message: %r' % (resp.code, (yield resp.content()), msg))
if self.config.get('message_queue') is not None:
yield self.ro_connector.publish_inbound(message)
yield self._increment_metric('inbound')
@inlineCallbacks
def store_and_forward_event(self, event):
'''Store the event in the message store, POST it to the correct
URL.'''
yield self._store_event(event)
yield self._forward_event(event)
yield self._count_event(event)
def _increment_metric(self, label):
return self.message_rate.increment(
self.channel_id, label, self.config['metric_window'])
def _count_event(self, event):
if event['event_type'] == 'ack':
return self._increment_metric('submitted')
if event['event_type'] == 'nack':
return self._increment_metric('rejected')
if event['event_type'] == 'delivery_report':
if event['delivery_status'] == 'pending':
return self._increment_metric('delivery_pending')
if event['delivery_status'] == 'failed':
return self._increment_metric('delivery_failed')
if event['delivery_status'] == 'delivered':
return self._increment_metric('delivery_succeeded')
def _store_event(self, event):
'''Stores the event in the message store'''
message_id = event['user_message_id']
if message_id is None:
logging.warning(
"Cannot store event, missing user_message_id: %r" % event)
else:
return self.outbounds.store_event(
self.channel_id, message_id, event)
@inlineCallbacks
def _forward_event(self, event):
'''Forward the event to the correct places.'''
yield self._forward_event_http(event)
yield self._forward_event_amqp(event)
@inlineCallbacks
def _forward_event_http(self, event):
'''POST the event to the correct URL'''
url = yield self._get_event_url(event)
if url is None:
return
(url, auth) = self._split_url_and_credentials(urlparse(url))
# Construct token auth headers if configured
auth_token = yield self._get_event_auth_token(event)
if auth_token:
headers = {
'Authorization': [
'Token %s' % (auth_token,)]
}
else:
headers = {}
msg = api_from_event(self.channel_id, event)
if msg['event_type'] is None:
logging.exception("Discarding unrecognised event %r" % (event,))
return
config = self.get_static_config()
resp = yield post(url, msg, timeout=config.event_url_timeout,
auth=auth, headers=headers)
if resp and request_failed(resp):
logging.exception(
'Error sending event, received HTTP code %r with body %r. '
'Event: %r' % (resp.code, (yield resp.content()), event))
def _forward_event_amqp(self, event):
'''Put the event on the correct queue.'''
if self.config.get('message_queue') is not None:
return self.ro_connector.publish_event(event)
def consume_ack(self, event):
return self.store_and_forward_event(event)
def consume_nack(self, event):
return self.store_and_forward_event(event)
def consume_delivery_report(self, event):
return self.store_and_forward_event(event)
def _split_url_and_credentials(self, url):
# Parse the basic auth username & password if available
username = url.username
password = url.password
if any([username, password]):
url = urlunparse((
url.scheme,
'%s%s' % (url.hostname, (':%s' % (url.port,)
if url.port is not None else '')),
url.path,
url.params,
url.query,
url.fragment,
))
auth = (username, password)
return (url, auth)
return (url.geturl(), None)
def _get_event_url(self, event):
msg_id = event['user_message_id']
if msg_id is not None:
return self.outbounds.load_event_url(self.channel_id, msg_id)
else:
logging.warning(
"Cannot find event URL, missing user_message_id: %r" % event)
def _get_event_auth_token(self, event):
msg_id = event['user_message_id']
if msg_id is not None:
return self.outbounds.load_event_auth_token(
self.channel_id, msg_id)
else:
logging.warning(
"Cannot find event auth, missing user_message_id: %r" % event)
class MessageForwardingWorker(ApplicationWorker):
'''This application worker consumes vumi messages placed on a configured
amqp queue, and sends them as HTTP requests with a JSON body to a
configured URL'''
CONFIG_CLASS = MessageForwardingConfig
@inlineCallbacks
def setup_application(self):
self.redis = yield TxRedisManager.from_config(
self.config['redis_manager'])
self.inbounds = InboundMessageStore(self.redis,
self.config['inbound_ttl'])
self.outbounds = OutboundMessageStore(self.redis,
self.config['outbound_ttl'])
self.message_rate = MessageRateStore(self.redis)
if self.config.get('message_queue') is not None:
self.ro_connector = yield self.setup_ro_connector(
self.config['message_queue'])
self.ro_connector.set_outbound_handler(self._publish_message)
@inlineCallbacks
def teardown_application(self):
if getattr(self, 'redis', None) is not None:
yield self.redis.close_manager()
@property
def channel_id(self):
return self.config['transport_name']
@inlineCallbacks
def consume_user_message(self, message):
'''Sends the vumi message as an HTTP request to the configured URL'''
yield self.inbounds.store_vumi_message(self.channel_id, message)
msg = api_from_message(message)
if self.config.get('mo_message_url') is not None:
config = self.get_static_config()
# Parse the basic auth username & password if available
username = config.mo_message_url.username
password = config.mo_message_url.password
if any([username, password]):
url = urlunparse((
config.mo_message_url.scheme,
'%s%s' %
(config.mo_message_url.hostname,
(':%s' % (config.mo_message_url.port, )
if config.mo_message_url.port is not None else '')),
config.mo_message_url.path,
config.mo_message_url.params,
config.mo_message_url.query,
config.mo_message_url.fragment,
))
auth = (username, password)
else:
url = config.mo_message_url.geturl()
auth = None
# Construct token auth headers if configured
auth_token = config.mo_message_url_auth_token
if auth_token:
headers = {'Authorization': ['Token %s' % (auth_token, )]}
else:
headers = {}
resp = yield post(url,
msg,
timeout=config.mo_message_url_timeout,
auth=auth,
headers=headers)
if resp and request_failed(resp):
logging.exception(
'Error sending message, received HTTP code %r with body %r'
'. Message: %r' % (resp.code, (yield resp.content()), msg))
if self.config.get('message_queue') is not None:
yield self.ro_connector.publish_inbound(message)
yield self._increment_metric('inbound')
@inlineCallbacks
def store_and_forward_event(self, event):
'''Store the event in the message store, POST it to the correct
URL.'''
yield self._store_event(event)
yield self._forward_event(event)
yield self._count_event(event)
def _increment_metric(self, label):
return self.message_rate.increment(self.channel_id, label,
self.config['metric_window'])
def _count_event(self, event):
if event['event_type'] == 'ack':
return self._increment_metric('submitted')
if event['event_type'] == 'nack':
return self._increment_metric('rejected')
if event['event_type'] == 'delivery_report':
if event['delivery_status'] == 'pending':
return self._increment_metric('delivery_pending')
if event['delivery_status'] == 'failed':
return self._increment_metric('delivery_failed')
if event['delivery_status'] == 'delivered':
return self._increment_metric('delivery_succeeded')
def _store_event(self, event):
'''Stores the event in the message store'''
message_id = event['user_message_id']
if message_id is None:
logging.warning("Cannot store event, missing user_message_id: %r" %
event)
else:
return self.outbounds.store_event(self.channel_id, message_id,
event)
@inlineCallbacks
def _forward_event(self, event):
'''Forward the event to the correct places.'''
yield self._forward_event_http(event)
yield self._forward_event_amqp(event)
@inlineCallbacks
def _forward_event_http(self, event):
'''POST the event to the correct URL'''
url = yield self._get_event_url(event)
if url is None:
return
msg = api_from_event(self.channel_id, event)
if msg['event_type'] is None:
logging.exception("Discarding unrecognised event %r" % (event, ))
return
config = self.get_static_config()
resp = yield post(url, msg, timeout=config.event_url_timeout)
if resp and request_failed(resp):
logging.exception(
'Error sending event, received HTTP code %r with body %r. '
'Event: %r' % (resp.code, (yield resp.content()), event))
def _forward_event_amqp(self, event):
'''Put the event on the correct queue.'''
if self.config.get('message_queue') is not None:
return self.ro_connector.publish_event(event)
def consume_ack(self, event):
return self.store_and_forward_event(event)
def consume_nack(self, event):
return self.store_and_forward_event(event)
def consume_delivery_report(self, event):
return self.store_and_forward_event(event)
def _get_event_url(self, event):
msg_id = event['user_message_id']
if msg_id is not None:
return self.outbounds.load_event_url(self.channel_id, msg_id)
else:
logging.warning(
"Cannot find event URL, missing user_message_id: %r" % event)
def create_store(self, ttl=60):
redis = yield self.get_redis()
store = InboundMessageStore(redis, ttl)
returnValue(store)
class MessageForwardingWorker(ApplicationWorker):
'''This application worker consumes vumi messages placed on a configured
amqp queue, and sends them as HTTP requests with a JSON body to a
configured URL'''
CONFIG_CLASS = MessageForwardingConfig
@inlineCallbacks
def setup_application(self):
self.redis = yield TxRedisManager.from_config(
self.config['redis_manager'])
self.inbounds = InboundMessageStore(
self.redis, self.config['inbound_ttl'])
self.outbounds = OutboundMessageStore(
self.redis, self.config['outbound_ttl'])
self.message_rate = MessageRateStore(self.redis)
if self.config.get('message_queue') is not None:
self.ro_connector = yield self.setup_ro_connector(
self.config['message_queue'])
self.ro_connector.set_outbound_handler(
self._publish_message)
@inlineCallbacks
def teardown_application(self):
yield self.redis.close_manager()
@property
def channel_id(self):
return self.config['transport_name']
@inlineCallbacks
def consume_user_message(self, message):
'''Sends the vumi message as an HTTP request to the configured URL'''
yield self.inbounds.store_vumi_message(self.channel_id, message)
msg = api_from_message(message)
if self.config.get('mo_message_url') is not None:
resp = yield post(self.config['mo_message_url'], msg)
if request_failed(resp):
logging.exception(
'Error sending message, received HTTP code %r with body %r'
'. Message: %r' % (resp.code, (yield resp.content()), msg))
if self.config.get('message_queue') is not None:
yield self.ro_connector.publish_inbound(message)
yield self._increment_metric('inbound')
@inlineCallbacks
def store_and_forward_event(self, event):
'''Store the event in the message store, POST it to the correct
URL.'''
yield self._store_event(event)
yield self._forward_event(event)
yield self._count_event(event)
def _increment_metric(self, label):
return self.message_rate.increment(
self.channel_id, label, self.config['metric_window'])
def _count_event(self, event):
if event['event_type'] == 'ack':
return self._increment_metric('submitted')
if event['event_type'] == 'nack':
return self._increment_metric('rejected')
if event['event_type'] == 'delivery_report':
if event['delivery_status'] == 'pending':
return self._increment_metric('delivery_pending')
if event['delivery_status'] == 'failed':
return self._increment_metric('delivery_failed')
if event['delivery_status'] == 'delivered':
return self._increment_metric('delivery_succeeded')
def _store_event(self, event):
'''Stores the event in the message store'''
message_id = event['user_message_id']
return self.outbounds.store_event(self.channel_id, message_id, event)
@inlineCallbacks
def _forward_event(self, event):
'''Forward the event to the correct places.'''
yield self._forward_event_http(event)
yield self._forward_event_amqp(event)
@inlineCallbacks
def _forward_event_http(self, event):
'''POST the event to the correct URL'''
url = yield self._get_event_url(event)
if url is None:
return
msg = api_from_event(self.channel_id, event)
if msg['event_type'] is None:
logging.exception("Discarding unrecognised event %r" % (event,))
return
resp = yield post(url, msg)
if request_failed(resp):
logging.exception(
'Error sending event, received HTTP code %r with body %r. '
'Event: %r' % (resp.code, (yield resp.content()), event))
def _forward_event_amqp(self, event):
'''Put the event on the correct queue.'''
if self.config.get('message_queue') is not None:
return self.ro_connector.publish_event(event)
def consume_ack(self, event):
return self.store_and_forward_event(event)
def consume_nack(self, event):
return self.store_and_forward_event(event)
def consume_delivery_report(self, event):
return self.store_and_forward_event(event)
def _get_event_url(self, event):
msg_id = event['user_message_id']
return self.outbounds.load_event_url(self.channel_id, msg_id)
class MessageForwardingWorker(ApplicationWorker):
'''This application worker consumes vumi messages placed on a configured
amqp queue, and sends them as HTTP requests with a JSON body to a
configured URL'''
CONFIG_CLASS = MessageForwardingConfig
@inlineCallbacks
def setup_application(self):
self.redis = yield TxRedisManager.from_config(
self.config['redis_manager'])
self.inbounds = InboundMessageStore(
self.redis, self.config['inbound_ttl'])
self.outbounds = OutboundMessageStore(
self.redis, self.config['outbound_ttl'])
@inlineCallbacks
def teardown_application(self):
yield self.redis.close_manager()
@property
def channel_id(self):
return self.config['transport_name']
@inlineCallbacks
def consume_user_message(self, message):
'''Sends the vumi message as an HTTP request to the configured URL'''
yield self.inbounds.store_vumi_message(self.channel_id, message)
msg = api_from_message(message)
resp = yield post(self.config['mo_message_url'], msg)
if request_failed(resp):
logging.exception(
'Error sending message, received HTTP code %r with body %r. '
'Message: %r' % (resp.code, (yield resp.content()), msg))
@inlineCallbacks
def forward_event(self, event):
url = yield self._get_event_url(event)
if url is None:
return
msg = api_from_event(self.channel_id, event)
if msg['event_type'] is None:
logging.exception("Discarding unrecognised event %r" % (event,))
return
resp = yield post(url, msg)
if request_failed(resp):
logging.exception(
'Error sending event, received HTTP code %r with body %r. '
'Event: %r' % (resp.code, (yield resp.content()), event))
def consume_ack(self, event):
return self.forward_event(event)
def consume_nack(self, event):
return self.forward_event(event)
def consume_delivery_report(self, event):
return self.forward_event(event)
def _get_event_url(self, event):
msg_id = event['user_message_id']
return self.outbounds.load_event_url(self.channel_id, msg_id)