def setup(self, redis=None, message_sender=None):
if redis is None:
redis = yield TxRedisManager.from_config(self.redis_config)
if message_sender is None:
message_sender = MessageSender('amqp-spec-0-8.xml',
self.amqp_config)
self.redis = redis
self.message_sender = message_sender
self.message_sender.setServiceParent(self.service)
self.inbounds = InboundMessageStore(self.redis,
self.config.inbound_message_ttl)
self.outbounds = OutboundMessageStore(self.redis,
self.config.outbound_message_ttl)
self.message_rate = MessageRateStore(self.redis)
self.plugins = []
for plugin_config in self.config.plugins:
cls = load_class_by_string(plugin_config['type'])
plugin = cls()
yield plugin.start_plugin(plugin_config, self.config)
self.plugins.append(plugin)
yield Channel.start_all_channels(self.redis, self.config, self.service,
self.plugins)
def setup_application(self):
self.redis = yield TxRedisManager.from_config(
self.config['redis_manager'])
self.inbounds = InboundMessageStore(self.redis,
self.config['inbound_ttl'])
self.outbounds = OutboundMessageStore(self.redis,
self.config['outbound_ttl'])
self.message_rate = MessageRateStore(self.redis)
if self.config.get('message_queue') is not None:
self.ro_connector = yield self.setup_ro_connector(
self.config['message_queue'])
self.ro_connector.set_outbound_handler(self._publish_message)
def setup(self, redis=None, message_sender=None):
if redis is None:
redis = yield TxRedisManager.from_config(self.redis_config)
if message_sender is None:
message_sender = MessageSender(
'amqp-spec-0-8.xml', self.amqp_config)
self.redis = redis
self.message_sender = message_sender
self.message_sender.setServiceParent(self.service)
self.inbounds = InboundMessageStore(
self.redis, self.config.inbound_message_ttl)
self.outbounds = OutboundMessageStore(
self.redis, self.config.outbound_message_ttl)
self.message_rate = MessageRateStore(self.redis)
self.plugins = []
for plugin_config in self.config.plugins:
cls = load_class_by_string(plugin_config['type'])
plugin = cls()
yield plugin.start_plugin(plugin_config, self.config)
self.plugins.append(plugin)
yield Channel.start_all_channels(
self.redis, self.config, self.service, self.plugins)
def __init__(self, redis_manager, config, properties, plugins=[], id=None):
'''Creates a new channel. ``redis_manager`` is the redis manager, from
which a sub manager is created using the channel id. If the channel id
is not supplied, a UUID one is generated. Call ``save`` to save the
channel data. It can be started using the ``start`` function.'''
self._properties = properties
self.redis = redis_manager
self.id = id
self.config = config
if self.id is None:
self.id = str(uuid.uuid4())
self.options = deepcopy(VumiOptions.default_vumi_options)
self.options.update(self.config.amqp)
self.transport_worker = None
self.application_worker = None
self.status_application_worker = None
self.sstore = StatusStore(self.redis)
self.plugins = plugins
self.message_rates = MessageRateStore(self.redis)
def setup_application(self):
self.redis = yield TxRedisManager.from_config(
self.config['redis_manager'])
self.inbounds = InboundMessageStore(
self.redis, self.config['inbound_ttl'])
self.outbounds = OutboundMessageStore(
self.redis, self.config['outbound_ttl'])
self.message_rate = MessageRateStore(self.redis)
if self.config.get('message_queue') is not None:
self.ro_connector = yield self.setup_ro_connector(
self.config['message_queue'])
self.ro_connector.set_outbound_handler(
self._publish_message)
class MessageForwardingWorker(ApplicationWorker):
'''This application worker consumes vumi messages placed on a configured
amqp queue, and sends them as HTTP requests with a JSON body to a
configured URL'''
CONFIG_CLASS = MessageForwardingConfig
@inlineCallbacks
def setup_application(self):
self.redis = yield TxRedisManager.from_config(
self.config['redis_manager'])
self.inbounds = InboundMessageStore(self.redis,
self.config['inbound_ttl'])
self.outbounds = OutboundMessageStore(self.redis,
self.config['outbound_ttl'])
self.message_rate = MessageRateStore(self.redis)
if self.config.get('message_queue') is not None:
self.ro_connector = yield self.setup_ro_connector(
self.config['message_queue'])
self.ro_connector.set_outbound_handler(self._publish_message)
@inlineCallbacks
def teardown_application(self):
yield self.redis.close_manager()
@property
def channel_id(self):
return self.config['transport_name']
@inlineCallbacks
def consume_user_message(self, message):
'''Sends the vumi message as an HTTP request to the configured URL'''
yield self.inbounds.store_vumi_message(self.channel_id, message)
msg = api_from_message(message)
if self.config.get('mo_message_url') is not None:
resp = yield post(self.config['mo_message_url'], msg)
if request_failed(resp):
logging.exception(
'Error sending message, received HTTP code %r with body %r'
'. Message: %r' % (resp.code, (yield resp.content()), msg))
if self.config.get('message_queue') is not None:
yield self.ro_connector.publish_inbound(message)
yield self._increment_metric('inbound')
@inlineCallbacks
def store_and_forward_event(self, event):
'''Store the event in the message store, POST it to the correct
URL.'''
yield self._store_event(event)
yield self._forward_event(event)
yield self._count_event(event)
def _increment_metric(self, label):
return self.message_rate.increment(self.channel_id, label,
self.config['metric_window'])
def _count_event(self, event):
if event['event_type'] == 'ack':
return self._increment_metric('submitted')
if event['event_type'] == 'nack':
return self._increment_metric('rejected')
if event['event_type'] == 'delivery_report':
if event['delivery_status'] == 'pending':
return self._increment_metric('delivery_pending')
if event['delivery_status'] == 'failed':
return self._increment_metric('delivery_failed')
if event['delivery_status'] == 'delivered':
return self._increment_metric('delivery_succeeded')
def _store_event(self, event):
'''Stores the event in the message store'''
message_id = event['user_message_id']
return self.outbounds.store_event(self.channel_id, message_id, event)
@inlineCallbacks
def _forward_event(self, event):
'''Forward the event to the correct places.'''
yield self._forward_event_http(event)
yield self._forward_event_amqp(event)
@inlineCallbacks
def _forward_event_http(self, event):
'''POST the event to the correct URL'''
url = yield self._get_event_url(event)
if url is None:
return
msg = api_from_event(self.channel_id, event)
if msg['event_type'] is None:
logging.exception("Discarding unrecognised event %r" % (event, ))
return
resp = yield post(url, msg)
if request_failed(resp):
logging.exception(
'Error sending event, received HTTP code %r with body %r. '
'Event: %r' % (resp.code, (yield resp.content()), event))
def _forward_event_amqp(self, event):
'''Put the event on the correct queue.'''
if self.config.get('message_queue') is not None:
return self.ro_connector.publish_event(event)
def consume_ack(self, event):
return self.store_and_forward_event(event)
def consume_nack(self, event):
return self.store_and_forward_event(event)
def consume_delivery_report(self, event):
return self.store_and_forward_event(event)
def _get_event_url(self, event):
msg_id = event['user_message_id']
return self.outbounds.load_event_url(self.channel_id, msg_id)
class Channel(object):
OUTBOUND_QUEUE = '%s.outbound'
APPLICATION_ID = 'application:%s'
STATUS_APPLICATION_ID = 'status:%s'
APPLICATION_CLS_NAME = 'junebug.workers.MessageForwardingWorker'
STATUS_APPLICATION_CLS_NAME = 'junebug.workers.ChannelStatusWorker'
JUNEBUG_LOGGING_SERVICE_CLS = JunebugLoggerService
def __init__(self, redis_manager, config, properties, plugins=[], id=None):
'''Creates a new channel. ``redis_manager`` is the redis manager, from
which a sub manager is created using the channel id. If the channel id
is not supplied, a UUID one is generated. Call ``save`` to save the
channel data. It can be started using the ``start`` function.'''
self._properties = properties
self.redis = redis_manager
self.id = id
self.config = config
if self.id is None:
self.id = str(uuid.uuid4())
self.options = deepcopy(VumiOptions.default_vumi_options)
self.options.update(self.config.amqp)
self.transport_worker = None
self.application_worker = None
self.status_application_worker = None
self.sstore = StatusStore(self.redis)
self.plugins = plugins
self.message_rates = MessageRateStore(self.redis)
@property
def application_id(self):
return self.APPLICATION_ID % (self.id,)
@property
def status_application_id(self):
return self.STATUS_APPLICATION_ID % (self.id,)
@property
def character_limit(self):
return self._properties.get('character_limit')
@inlineCallbacks
def start(self, service, transport_worker=None):
'''Starts the relevant workers for the channel. ``service`` is the
parent of under which the workers should be started.'''
self._start_transport(service, transport_worker)
self._start_application(service)
self._start_status_application(service)
for plugin in self.plugins:
yield plugin.channel_started(self)
@inlineCallbacks
def stop(self):
'''Stops the relevant workers for the channel'''
yield self._stop_application()
yield self._stop_status_application()
yield self._stop_transport()
for plugin in self.plugins:
yield plugin.channel_stopped(self)
@inlineCallbacks
def save(self):
'''Saves the channel data into redis.'''
properties = json.dumps(self._properties)
channel_redis = yield self.redis.sub_manager(self.id)
yield channel_redis.set('properties', properties)
yield self.redis.sadd('channels', self.id)
@inlineCallbacks
def update(self, properties):
'''Updates the channel configuration, saves the updated configuration,
and (if needed) restarts the channel with the new configuration.
Returns the updated configuration and status.'''
self._properties.update(properties)
yield self.save()
service = self.transport_worker.parent
# Only restart if the channel config has changed
if 'config' in properties:
yield self._stop_transport()
yield self._start_transport(service)
if 'mo_url' in properties:
yield self._stop_application()
yield self._start_application(service)
returnValue((yield self.status()))
@inlineCallbacks
def delete(self):
'''Removes the channel data from redis'''
channel_redis = yield self.redis.sub_manager(self.id)
yield channel_redis.delete('properties')
yield self.redis.srem('channels', self.id)
@inlineCallbacks
def status(self):
'''Returns a dict with the configuration and status of the channel'''
status = deepcopy(self._properties)
status['id'] = self.id
status['status'] = yield self._get_status()
returnValue(status)
def _get_message_rate(self, label):
return self.message_rates.get_messages_per_second(
self.id, label, self.config.metric_window)
@inlineCallbacks
def _get_status(self):
components = yield self.sstore.get_statuses(self.id)
components = dict(
(k, api_from_status(self.id, v)) for k, v in components.iteritems()
)
status_values = {
'down': 0,
'degraded': 1,
'ok': 2,
}
try:
status = min(
(c['status'] for c in components.values()),
key=status_values.get)
except ValueError:
# No statuses
status = None
returnValue({
'components': components,
'status': status,
'inbound_message_rate': (
yield self._get_message_rate('inbound')),
'outbound_message_rate': (
yield self._get_message_rate('outbound')),
'submitted_event_rate': (
yield self._get_message_rate('submitted')),
'rejected_event_rate': (
yield self._get_message_rate('rejected')),
'delivery_succeeded_rate': (
yield self._get_message_rate('delivery_succeeded')),
'delivery_failed_rate': (
yield self._get_message_rate('delivery_failed')),
'delivery_pending_rate': (
yield self._get_message_rate('delivery_pending')),
})
@classmethod
@inlineCallbacks
def from_id(cls, redis, config, id, parent, plugins=[]):
'''Creates a channel by loading the data from redis, given the
channel's id, and the parent service of the channel'''
channel_redis = yield redis.sub_manager(id)
properties = yield channel_redis.get('properties')
if properties is None:
raise ChannelNotFound()
properties = json.loads(properties)
obj = cls(redis, config, properties, plugins, id=id)
obj._restore(parent)
returnValue(obj)
@classmethod
@inlineCallbacks
def get_all(cls, redis):
'''Returns a set of keys of all of the channels'''
channels = yield redis.smembers('channels')
returnValue(channels)
@classmethod
@inlineCallbacks
def start_all_channels(cls, redis, config, parent, plugins=[]):
'''Ensures that all of the stored channels are running'''
for id in (yield cls.get_all(redis)):
if id not in parent.namedServices:
properties = json.loads((
yield redis.get('%s:properties' % id)))
channel = cls(redis, config, properties, plugins, id=id)
yield channel.start(parent)
@inlineCallbacks
def send_message(self, sender, outbounds, msg):
'''Sends a message.'''
event_url = msg.get('event_url')
msg = message_from_api(self.id, msg)
msg = TransportUserMessage.send(**msg)
msg = yield self._send_message(sender, outbounds, event_url, msg)
returnValue(api_from_message(msg))
@inlineCallbacks
def send_reply_message(self, sender, outbounds, inbounds, msg):
'''Sends a reply message.'''
in_msg = yield inbounds.load_vumi_message(self.id, msg['reply_to'])
if in_msg is None:
raise MessageNotFound(
"Inbound message with id %s not found" % (msg['reply_to'],))
event_url = msg.get('event_url')
msg = message_from_api(self.id, msg)
msg = in_msg.reply(**msg)
msg = yield self._send_message(sender, outbounds, event_url, msg)
returnValue(api_from_message(msg))
def get_logs(self, n):
'''Returns the last `n` logs. If `n` is greater than the configured
limit, only returns the configured limit amount of logs. If `n` is
None, returns the configured limit amount of logs.'''
if n is None:
n = self.config.max_logs
n = min(n, self.config.max_logs)
logfile = self.transport_worker.getServiceNamed(
'Junebug Worker Logger').logfile
return read_logs(logfile, n)
@property
def _transport_config(self):
config = self._properties['config']
config = self._convert_unicode(config)
config['transport_name'] = self.id
config['worker_name'] = self.id
config['publish_status'] = True
return config
@property
def _application_config(self):
return {
'transport_name': self.id,
'mo_message_url': self._properties['mo_url'],
'redis_manager': self.config.redis,
'inbound_ttl': self.config.inbound_message_ttl,
'outbound_ttl': self.config.outbound_message_ttl,
'metric_window': self.config.metric_window,
}
@property
def _status_application_config(self):
return {
'redis_manager': self.config.redis,
'channel_id': self.id,
'status_url': self._properties.get('status_url'),
}
@property
def _available_transports(self):
if self.config.replace_channels:
return self.config.channels
else:
channels = {}
channels.update(transports)
channels.update(self.config.channels)
return channels
@property
def _transport_cls_name(self):
cls_name = self._available_transports.get(self._properties.get('type'))
if cls_name is None:
raise InvalidChannelType(
'Invalid channel type %r, must be one of: %s' % (
self._properties.get('type'),
', '.join(self._available_transports.keys())))
return cls_name
def _start_transport(self, service, transport_worker=None):
# transport_worker parameter is for testing, if it is None,
# create the transport worker
if transport_worker is None:
transport_worker = self._create_transport()
transport_worker.setName(self.id)
logging_service = self._create_junebug_logger_service()
transport_worker.addService(logging_service)
transport_worker.setServiceParent(service)
self.transport_worker = transport_worker
def _start_application(self, service):
worker = self._create_application()
worker.setName(self.application_id)
worker.setServiceParent(service)
self.application_worker = worker
def _start_status_application(self, service):
worker = self._create_status_application()
worker.setName(self.status_application_id)
worker.setServiceParent(service)
self.status_application_worker = worker
def _create_transport(self):
return self._create_worker(
self._transport_cls_name,
self._transport_config)
def _create_application(self):
return self._create_worker(
self.APPLICATION_CLS_NAME,
self._application_config)
def _create_status_application(self):
return self._create_worker(
self.STATUS_APPLICATION_CLS_NAME,
self._status_application_config)
def _create_junebug_logger_service(self):
return self.JUNEBUG_LOGGING_SERVICE_CLS(
self.id, self.config.logging_path, self.config.log_rotate_size,
self.config.max_log_files)
def _create_worker(self, cls_name, config):
creator = WorkerCreator(self.options)
worker = creator.create_worker(cls_name, config)
return worker
@inlineCallbacks
def _stop_transport(self):
if self.transport_worker is not None:
yield self.transport_worker.disownServiceParent()
self.transport_worker = None
@inlineCallbacks
def _stop_application(self):
if self.application_worker is not None:
yield self.application_worker.disownServiceParent()
self.application_worker = None
@inlineCallbacks
def _stop_status_application(self):
if self.status_application_worker is not None:
yield self.status_application_worker.disownServiceParent()
self.status_application_worker = None
def _restore(self, service):
self.transport_worker = service.getServiceNamed(self.id)
self.application_worker = service.getServiceNamed(self.application_id)
self.status_application_worker = service.getServiceNamed(
self.status_application_id)
def _convert_unicode(self, data):
# Twisted doesn't like it when we give unicode in for config things
if isinstance(data, basestring):
return str(data)
elif isinstance(data, collections.Mapping):
return dict(map(self._convert_unicode, data.iteritems()))
elif isinstance(data, collections.Iterable):
return type(data)(map(self._convert_unicode, data))
else:
return data
def _check_character_limit(self, content):
count = len(content)
if (self.character_limit is not None and count > self.character_limit):
raise MessageTooLong(
'Message content %r is of length %d, which is greater than the'
' character limit of %d' % (
content, count, self.character_limit)
)
@inlineCallbacks
def _send_message(self, sender, outbounds, event_url, msg):
self._check_character_limit(msg['content'])
if event_url is not None:
yield outbounds.store_event_url(
self.id, msg['message_id'], event_url)
queue = self.OUTBOUND_QUEUE % (self.id,)
msg = yield sender.send_message(msg, routing_key=queue)
returnValue(msg)
class MessageForwardingWorker(ApplicationWorker):
'''This application worker consumes vumi messages placed on a configured
amqp queue, and sends them as HTTP requests with a JSON body to a
configured URL'''
CONFIG_CLASS = MessageForwardingConfig
@inlineCallbacks
def setup_application(self):
self.redis = yield TxRedisManager.from_config(
self.config['redis_manager'])
self.inbounds = InboundMessageStore(
self.redis, self.config['inbound_ttl'])
self.outbounds = OutboundMessageStore(
self.redis, self.config['outbound_ttl'])
self.message_rate = MessageRateStore(self.redis)
if self.config.get('message_queue') is not None:
self.ro_connector = yield self.setup_ro_connector(
self.config['message_queue'])
self.ro_connector.set_outbound_handler(
self._publish_message)
@inlineCallbacks
def teardown_application(self):
if getattr(self, 'redis', None) is not None:
yield self.redis.close_manager()
@property
def channel_id(self):
return self.config['transport_name']
@inlineCallbacks
def consume_user_message(self, message):
'''Sends the vumi message as an HTTP request to the configured URL'''
yield self.inbounds.store_vumi_message(self.channel_id, message)
msg = api_from_message(message)
if self.config.get('mo_message_url') is not None:
config = self.get_static_config()
(url, auth) = self._split_url_and_credentials(
config.mo_message_url)
# Construct token auth headers if configured
auth_token = config.mo_message_url_auth_token
if auth_token:
headers = {
'Authorization': [
'Token %s' % (auth_token,)]
}
else:
headers = {}
resp = yield post(url, msg,
timeout=config.mo_message_url_timeout,
auth=auth, headers=headers)
if resp and request_failed(resp):
logging.exception(
'Error sending message, received HTTP code %r with body %r'
'. Message: %r' % (resp.code, (yield resp.content()), msg))
if self.config.get('message_queue') is not None:
yield self.ro_connector.publish_inbound(message)
yield self._increment_metric('inbound')
@inlineCallbacks
def store_and_forward_event(self, event):
'''Store the event in the message store, POST it to the correct
URL.'''
yield self._store_event(event)
yield self._forward_event(event)
yield self._count_event(event)
def _increment_metric(self, label):
return self.message_rate.increment(
self.channel_id, label, self.config['metric_window'])
def _count_event(self, event):
if event['event_type'] == 'ack':
return self._increment_metric('submitted')
if event['event_type'] == 'nack':
return self._increment_metric('rejected')
if event['event_type'] == 'delivery_report':
if event['delivery_status'] == 'pending':
return self._increment_metric('delivery_pending')
if event['delivery_status'] == 'failed':
return self._increment_metric('delivery_failed')
if event['delivery_status'] == 'delivered':
return self._increment_metric('delivery_succeeded')
def _store_event(self, event):
'''Stores the event in the message store'''
message_id = event['user_message_id']
if message_id is None:
logging.warning(
"Cannot store event, missing user_message_id: %r" % event)
else:
return self.outbounds.store_event(
self.channel_id, message_id, event)
@inlineCallbacks
def _forward_event(self, event):
'''Forward the event to the correct places.'''
yield self._forward_event_http(event)
yield self._forward_event_amqp(event)
@inlineCallbacks
def _forward_event_http(self, event):
'''POST the event to the correct URL'''
url = yield self._get_event_url(event)
if url is None:
return
(url, auth) = self._split_url_and_credentials(urlparse(url))
# Construct token auth headers if configured
auth_token = yield self._get_event_auth_token(event)
if auth_token:
headers = {
'Authorization': [
'Token %s' % (auth_token,)]
}
else:
headers = {}
msg = api_from_event(self.channel_id, event)
if msg['event_type'] is None:
logging.exception("Discarding unrecognised event %r" % (event,))
return
config = self.get_static_config()
resp = yield post(url, msg, timeout=config.event_url_timeout,
auth=auth, headers=headers)
if resp and request_failed(resp):
logging.exception(
'Error sending event, received HTTP code %r with body %r. '
'Event: %r' % (resp.code, (yield resp.content()), event))
def _forward_event_amqp(self, event):
'''Put the event on the correct queue.'''
if self.config.get('message_queue') is not None:
return self.ro_connector.publish_event(event)
def consume_ack(self, event):
return self.store_and_forward_event(event)
def consume_nack(self, event):
return self.store_and_forward_event(event)
def consume_delivery_report(self, event):
return self.store_and_forward_event(event)
def _split_url_and_credentials(self, url):
# Parse the basic auth username & password if available
username = url.username
password = url.password
if any([username, password]):
url = urlunparse((
url.scheme,
'%s%s' % (url.hostname, (':%s' % (url.port,)
if url.port is not None else '')),
url.path,
url.params,
url.query,
url.fragment,
))
auth = (username, password)
return (url, auth)
return (url.geturl(), None)
def _get_event_url(self, event):
msg_id = event['user_message_id']
if msg_id is not None:
return self.outbounds.load_event_url(self.channel_id, msg_id)
else:
logging.warning(
"Cannot find event URL, missing user_message_id: %r" % event)
def _get_event_auth_token(self, event):
msg_id = event['user_message_id']
if msg_id is not None:
return self.outbounds.load_event_auth_token(
self.channel_id, msg_id)
else:
logging.warning(
"Cannot find event auth, missing user_message_id: %r" % event)
class JunebugApi(object):
app = Klein()
def __init__(self, service, config):
self.service = service
self.redis_config = config.redis
self.amqp_config = config.amqp
self.config = config
@inlineCallbacks
def setup(self, redis=None, message_sender=None):
if redis is None:
redis = yield TxRedisManager.from_config(self.redis_config)
if message_sender is None:
message_sender = MessageSender(
'amqp-spec-0-8.xml', self.amqp_config)
self.redis = redis
self.message_sender = message_sender
self.message_sender.setServiceParent(self.service)
self.inbounds = InboundMessageStore(
self.redis, self.config.inbound_message_ttl)
self.outbounds = OutboundMessageStore(
self.redis, self.config.outbound_message_ttl)
self.message_rate = MessageRateStore(self.redis)
self.router_store = RouterStore(self.redis)
self.plugins = []
for plugin_config in self.config.plugins:
cls = load_class_by_string(plugin_config['type'])
plugin = cls()
yield plugin.start_plugin(plugin_config, self.config)
self.plugins.append(plugin)
yield Channel.start_all_channels(
self.redis, self.config, self.service, self.plugins)
yield Router.start_all_routers(self)
if self.config.rabbitmq_management_interface:
self.rabbitmq_management_client = RabbitmqManagementClient(
self.config.rabbitmq_management_interface,
self.amqp_config['username'],
self.amqp_config['password'])
@inlineCallbacks
def teardown(self):
yield self.redis.close_manager()
for plugin in self.plugins:
yield plugin.stop_plugin()
@app.handle_errors(JunebugError)
def generic_junebug_error(self, request, failure):
return response(request, failure.value.description, {
'errors': [{
'type': failure.value.name,
'message': failure.getErrorMessage(),
}]
}, code=failure.value.code)
@app.handle_errors(HTTPException)
def http_error(self, request, failure):
error = {
'code': failure.value.code,
'type': failure.value.name,
'message': failure.getErrorMessage(),
}
if getattr(failure.value, 'new_url', None) is not None:
request.setHeader('Location', failure.value.new_url)
error['new_url'] = failure.value.new_url
return response(request, failure.value.description, {
'errors': [error],
}, code=failure.value.code)
@app.handle_errors
def generic_error(self, request, failure):
log.err(failure)
return response(request, 'generic error', {
'errors': [{
'type': failure.type.__name__,
'message': failure.getErrorMessage(),
}]
}, code=http.INTERNAL_SERVER_ERROR)
@app.route('/channels/', methods=['GET'])
@inlineCallbacks
def get_channel_list(self, request):
'''List all channels'''
ids = yield Channel.get_all(self.redis)
returnValue(response(request, 'channels listed', sorted(ids)))
@app.route('/channels/', methods=['POST'])
@json_body
@validate(
body_schema({
'type': 'object',
'properties': {
'type': {'type': 'string'},
'label': {'type': 'string'},
'config': {'type': 'object'},
'metadata': {'type': 'object'},
'status_url': {'type': 'string'},
'mo_url': {'type': 'string'},
'mo_url_auth_token': {'type': 'string'},
'amqp_queue': {'type': 'string'},
'rate_limit_count': {
'type': 'integer',
'minimum': 0,
},
'rate_limit_window': {
'type': 'integer',
'minimum': 0,
},
'character_limit': {
'type': 'integer',
'minimum': 0,
},
},
'required': ['type', 'config'],
}))
@inlineCallbacks
def create_channel(self, request, body):
'''Create a channel'''
channel = Channel(
self.redis, self.config, body, self.plugins)
yield channel.start(self.service)
yield channel.save()
returnValue(response(
request, 'channel created', (yield channel.status()),
code=http.CREATED))
@app.route('/channels/<string:channel_id>', methods=['GET'])
@inlineCallbacks
def get_channel(self, request, channel_id):
'''Return the channel configuration and a nested status object'''
channel = yield Channel.from_id(
self.redis, self.config, channel_id, self.service, self.plugins)
resp = yield channel.status()
returnValue(response(
request, 'channel found', resp))
@app.route('/channels/<string:channel_id>', methods=['POST'])
@json_body
@validate(
body_schema({
'type': 'object',
'properties': {
'type': {'type': 'string'},
'label': {'type': 'string'},
'config': {'type': 'object'},
'metadata': {'type': 'object'},
'status_url': {'type': ['string', 'null']},
'mo_url': {'type': ['string', 'null']},
'rate_limit_count': {
'type': 'integer',
'minimum': 0,
},
'rate_limit_window': {
'type': 'integer',
'minimum': 0,
},
'character_limit': {
'type': 'integer',
'minimum': 0,
},
},
}))
@inlineCallbacks
def modify_channel(self, request, body, channel_id):
'''Mondify the channel configuration'''
channel = yield Channel.from_id(
self.redis, self.config, channel_id, self.service, self.plugins)
resp = yield channel.update(body)
returnValue(response(
request, 'channel updated', resp))
@app.route('/channels/<string:channel_id>', methods=['DELETE'])
@inlineCallbacks
def delete_channel(self, request, channel_id):
'''Delete the channel'''
channel = yield Channel.from_id(
self.redis, self.config, channel_id, self.service, self.plugins)
yield channel.stop()
yield channel.delete()
returnValue(response(
request, 'channel deleted', {}))
@app.route('/channels/<string:channel_id>/restart', methods=['POST'])
@inlineCallbacks
def restart_channel(self, request, channel_id):
'''Restart a channel.'''
channel = yield Channel.from_id(
self.redis, self.config, channel_id, self.service, self.plugins)
yield channel.stop()
yield channel.start(self.service)
returnValue(response(request, 'channel restarted', {}))
@app.route('/channels/<string:channel_id>/logs', methods=['GET'])
@inlineCallbacks
def get_logs(self, request, channel_id):
'''Get the last N logs for a channel, sorted reverse
chronologically.'''
n = request.args.get('n', None)
if n is not None:
n = int(n[0])
channel = yield Channel.from_id(
self.redis, self.config, channel_id, self.service, self.plugins)
logs = yield channel.get_logs(n)
returnValue(response(request, 'logs retrieved', logs))
@app.route('/channels/<string:channel_id>/messages/', methods=['POST'])
@json_body
@validate(
body_schema({
'type': 'object',
'properties': {
'to': {'type': 'string'},
'from': {'type': ['string', 'null']},
'group': {'type': ['string', 'null']},
'reply_to': {'type': 'string'},
'content': {'type': ['string', 'null']},
'event_url': {'type': 'string'},
'event_auth_token': {'type': 'string'},
'priority': {'type': 'string'},
'channel_data': {'type': 'object'},
},
'required': ['content'],
'additionalProperties': False,
}))
@inlineCallbacks
def send_message(self, request, body, channel_id):
'''Send an outbound (mobile terminated) message'''
if 'to' not in body and 'reply_to' not in body:
raise ApiUsageError(
'Either "to" or "reply_to" must be specified')
channel = yield Channel.from_id(
self.redis, self.config, channel_id, self.service, self.plugins)
if 'reply_to' in body:
msg = yield channel.send_reply_message(
self.message_sender, self.outbounds, self.inbounds, body,
allow_expired_replies=self.config.allow_expired_replies)
else:
msg = yield channel.send_message(
self.message_sender, self.outbounds, body)
yield self.message_rate.increment(
channel_id, 'outbound', self.config.metric_window)
returnValue(response(
request, 'message submitted', msg, code=http.CREATED))
@app.route(
'/channels/<string:channel_id>/messages/<string:message_id>',
methods=['GET'])
@inlineCallbacks
def get_message_status(self, request, channel_id, message_id):
'''Retrieve the status of a message'''
events = yield self.outbounds.load_all_events(channel_id, message_id)
events = sorted(
(api_from_event(channel_id, e) for e in events),
key=lambda e: e['timestamp'])
last_event = events[-1] if events else None
last_event_type = last_event['event_type'] if last_event else None
last_event_timestamp = last_event['timestamp'] if last_event else None
returnValue(response(request, 'message status', {
'id': message_id,
'last_event_type': last_event_type,
'last_event_timestamp': last_event_timestamp,
'events': events,
}))
@app.route('/routers/', methods=['GET'])
def get_router_list(self, request):
"""List all routers"""
d = Router.get_all(self.router_store)
d.addCallback(partial(response, request, 'routers retrieved'))
return d
@app.route('/routers/', methods=['POST'])
@json_body
@validate(
body_schema({
'type': 'object',
'properties': {
'type': {'type': 'string'},
'label': {'type': 'string'},
'config': {'type': 'object'},
'metadata': {'type': 'object'},
},
'additionalProperties': False,
'required': ['type', 'config'],
}))
@inlineCallbacks
def create_router(self, request, body):
"""Create a new router"""
router = Router(self, body)
yield router.validate_config()
router.start(self.service)
yield router.save()
returnValue(response(
request,
'router created',
(yield router.status()),
code=http.CREATED
))
@app.route('/routers/<string:router_id>', methods=['GET'])
def get_router(self, request, router_id):
"""Get the configuration details and status of a specific router"""
d = Router.from_id(self, router_id)
d.addCallback(lambda router: router.status())
d.addCallback(partial(response, request, 'router found'))
return d
@app.route('/routers/<string:router_id>', methods=['PUT'])
@json_body
@validate(
body_schema({
'type': 'object',
'properties': {
'type': {'type': 'string'},
'label': {'type': 'string'},
'config': {'type': 'object'},
'metadata': {'type': 'object'},
},
'additionalProperties': False,
'required': ['type', 'config'],
}))
@inlineCallbacks
def replace_router_config(self, request, body, router_id):
"""Replace the router config with the one specified"""
router = yield Router.from_id(self, router_id)
for field in ['type', 'label', 'config', 'metadata']:
router.router_config.pop(field, None)
router.router_config.update(body)
yield router.validate_config()
# Stop and start the router for the worker to get the new config
yield router.stop()
router.start(self.service)
yield router.save()
returnValue(response(
request, 'router updated', (yield router.status())))
@app.route('/routers/<string:router_id>', methods=['PATCH'])
@json_body
@validate(
body_schema({
'type': 'object',
'properties': {
'type': {'type': 'string'},
'label': {'type': 'string'},
'config': {'type': 'object'},
'metadata': {'type': 'object'},
},
'additionalProperties': False,
'required': [],
}))
@inlineCallbacks
def update_router_config(self, request, body, router_id):
"""Update the router config with the one specified"""
router = yield Router.from_id(self, router_id)
router.router_config.update(body)
yield router.validate_config()
# Stop and start the router for the worker to get the new config
yield router.stop()
router.start(self.service)
yield router.save()
returnValue(response(
request, 'router updated', (yield router.status())))
@app.route('/routers/<string:router_id>', methods=['DELETE'])
@inlineCallbacks
def delete_router(self, request, router_id):
router = yield Router.from_id(self, router_id)
yield router.stop()
yield router.delete()
returnValue(response(request, 'router deleted', {}))
@app.route('/routers/<string:router_id>/destinations/', methods=['POST'])
@json_body
@validate(
body_schema({
'type': 'object',
'properties': {
'label': {'type': 'string'},
'config': {'type': 'object'},
'metadata': {'type': 'object'},
'mo_url': {'type': 'string'},
'mo_url_token': {'type': 'string'},
'amqp_queue': {'type': 'string'},
'character_limit': {
'type': 'integer',
'minimum': 0,
},
},
'additionalProperties': False,
'required': ['config'],
}))
@inlineCallbacks
def create_router_destination(self, request, body, router_id):
"""Create a new destination for the router"""
router = yield Router.from_id(self, router_id)
yield router.validate_destination_config(body['config'])
destination = router.add_destination(body)
yield router.stop()
router.start(self.service)
yield destination.save()
returnValue(response(
request, 'destination created', (yield destination.status()),
code=http.CREATED
))
@app.route('/routers/<string:router_id>/destinations/', methods=['GET'])
def get_router_destination_list(self, request, router_id):
"""Get the list of destinations for a router"""
d = Router.from_id(self, router_id)
d.addCallback(lambda router: router.get_destination_list())
d.addCallback(partial(response, request, 'destinations retrieved'))
return d
@app.route(
'/routers/<string:router_id>/destinations/<string:destination_id>',
methods=['GET'])
@inlineCallbacks
def get_destination(self, request, router_id, destination_id):
"""Get the config and status of a destination"""
router = yield Router.from_id(self, router_id)
destination = router.get_destination(destination_id)
returnValue(response(
request, 'destination found', (yield destination.status())
))
@app.route(
'/routers/<string:router_id>/destinations/<string:destination_id>',
methods=['PUT'])
@json_body
@validate(
body_schema({
'type': 'object',
'properties': {
'label': {'type': 'string'},
'config': {'type': 'object'},
'metadata': {'type': 'object'},
'mo_url': {'type': 'string'},
'mo_url_token': {'type': 'string'},
'amqp_queue': {'type': 'string'},
'character_limit': {
'type': 'integer',
'minimum': 0,
},
},
'additionalProperties': False,
'required': ['config'],
}))
@inlineCallbacks
def replace_router_destination(
self, request, body, router_id, destination_id):
"""Replace the config of a router destination"""
router = yield Router.from_id(self, router_id)
yield router.validate_destination_config(body['config'])
destination = router.get_destination(destination_id)
destination.destination_config = body
destination.destination_config['id'] = destination_id
# Stop and start the router for the worker to get the new config
yield router.stop()
router.start(self.service)
yield destination.save()
returnValue(response(
request, 'destination updated', (yield destination.status())))
@app.route(
'/routers/<string:router_id>/destinations/<string:destination_id>',
methods=['PATCH'])
@json_body
@validate(
body_schema({
'type': 'object',
'properties': {
'label': {'type': 'string'},
'config': {'type': 'object'},
'metadata': {'type': 'object'},
'mo_url': {'type': 'string'},
'mo_url_token': {'type': 'string'},
'amqp_queue': {'type': 'string'},
'character_limit': {
'type': 'integer',
'minimum': 0,
},
},
'additionalProperties': False,
'required': [],
}))
@inlineCallbacks
def update_router_destination(
self, request, body, router_id, destination_id):
"""Update the config of a router destination"""
router = yield Router.from_id(self, router_id)
if 'config' in body:
yield router.validate_destination_config(body['config'])
destination = router.get_destination(destination_id)
destination.destination_config.update(body)
# Stop and start the router for the worker to get the new config
yield router.stop()
router.start(self.service)
yield destination.save()
returnValue(response(
request, 'destination updated', (yield destination.status())))
@app.route(
'/routers/<string:router_id>/destinations/<string:destination_id>',
methods=['DELETE'])
@inlineCallbacks
def delete_router_destination(self, request, router_id, destination_id):
"""Delete and stop the router destination"""
router = yield Router.from_id(self, router_id)
destination = router.get_destination(destination_id)
yield router.stop()
yield destination.delete()
router.start(self.service)
returnValue(response(request, 'destination deleted', {}))
@app.route('/health', methods=['GET'])
def health_status(self, request):
if self.config.rabbitmq_management_interface:
def get_queues(channel_ids):
gets = []
for channel_id in channel_ids:
for sub in ['inbound', 'outbound', 'event']:
queue_name = "%s.%s" % (channel_id, sub)
get = self.rabbitmq_management_client.get_queue(
self.amqp_config['vhost'], queue_name)
gets.append(get)
return gets
def return_queue_results(results):
queues = []
stuck = False
for result in results:
queue = result[1]
if ('messages' in queue):
details = {
'name': queue['name'],
'stuck': False,
'messages': queue.get('messages'),
'rate': queue['messages_details']['rate']
}
if (details['messages'] > 0 and details['rate'] == 0):
stuck = True
details['stuck'] = True
queues.append(details)
status = 'queues ok'
code = http.OK
if stuck:
status = "queues stuck"
code = http.INTERNAL_SERVER_ERROR
return response(request, status, queues, code=code)
d = Channel.get_all(self.redis)
d.addCallback(get_queues)
d.addCallback(defer.DeferredList)
d.addCallback(return_queue_results)
return d
else:
return response(request, 'health ok', {})
class MessageForwardingWorker(ApplicationWorker):
'''This application worker consumes vumi messages placed on a configured
amqp queue, and sends them as HTTP requests with a JSON body to a
configured URL'''
CONFIG_CLASS = MessageForwardingConfig
@inlineCallbacks
def setup_application(self):
self.redis = yield TxRedisManager.from_config(
self.config['redis_manager'])
self.inbounds = InboundMessageStore(self.redis,
self.config['inbound_ttl'])
self.outbounds = OutboundMessageStore(self.redis,
self.config['outbound_ttl'])
self.message_rate = MessageRateStore(self.redis)
if self.config.get('message_queue') is not None:
self.ro_connector = yield self.setup_ro_connector(
self.config['message_queue'])
self.ro_connector.set_outbound_handler(self._publish_message)
@inlineCallbacks
def teardown_application(self):
if getattr(self, 'redis', None) is not None:
yield self.redis.close_manager()
@property
def channel_id(self):
return self.config['transport_name']
@inlineCallbacks
def consume_user_message(self, message):
'''Sends the vumi message as an HTTP request to the configured URL'''
yield self.inbounds.store_vumi_message(self.channel_id, message)
msg = api_from_message(message)
if self.config.get('mo_message_url') is not None:
config = self.get_static_config()
# Parse the basic auth username & password if available
username = config.mo_message_url.username
password = config.mo_message_url.password
if any([username, password]):
url = urlunparse((
config.mo_message_url.scheme,
'%s%s' %
(config.mo_message_url.hostname,
(':%s' % (config.mo_message_url.port, )
if config.mo_message_url.port is not None else '')),
config.mo_message_url.path,
config.mo_message_url.params,
config.mo_message_url.query,
config.mo_message_url.fragment,
))
auth = (username, password)
else:
url = config.mo_message_url.geturl()
auth = None
# Construct token auth headers if configured
auth_token = config.mo_message_url_auth_token
if auth_token:
headers = {'Authorization': ['Token %s' % (auth_token, )]}
else:
headers = {}
resp = yield post(url,
msg,
timeout=config.mo_message_url_timeout,
auth=auth,
headers=headers)
if resp and request_failed(resp):
logging.exception(
'Error sending message, received HTTP code %r with body %r'
'. Message: %r' % (resp.code, (yield resp.content()), msg))
if self.config.get('message_queue') is not None:
yield self.ro_connector.publish_inbound(message)
yield self._increment_metric('inbound')
@inlineCallbacks
def store_and_forward_event(self, event):
'''Store the event in the message store, POST it to the correct
URL.'''
yield self._store_event(event)
yield self._forward_event(event)
yield self._count_event(event)
def _increment_metric(self, label):
return self.message_rate.increment(self.channel_id, label,
self.config['metric_window'])
def _count_event(self, event):
if event['event_type'] == 'ack':
return self._increment_metric('submitted')
if event['event_type'] == 'nack':
return self._increment_metric('rejected')
if event['event_type'] == 'delivery_report':
if event['delivery_status'] == 'pending':
return self._increment_metric('delivery_pending')
if event['delivery_status'] == 'failed':
return self._increment_metric('delivery_failed')
if event['delivery_status'] == 'delivered':
return self._increment_metric('delivery_succeeded')
def _store_event(self, event):
'''Stores the event in the message store'''
message_id = event['user_message_id']
if message_id is None:
logging.warning("Cannot store event, missing user_message_id: %r" %
event)
else:
return self.outbounds.store_event(self.channel_id, message_id,
event)
@inlineCallbacks
def _forward_event(self, event):
'''Forward the event to the correct places.'''
yield self._forward_event_http(event)
yield self._forward_event_amqp(event)
@inlineCallbacks
def _forward_event_http(self, event):
'''POST the event to the correct URL'''
url = yield self._get_event_url(event)
if url is None:
return
msg = api_from_event(self.channel_id, event)
if msg['event_type'] is None:
logging.exception("Discarding unrecognised event %r" % (event, ))
return
config = self.get_static_config()
resp = yield post(url, msg, timeout=config.event_url_timeout)
if resp and request_failed(resp):
logging.exception(
'Error sending event, received HTTP code %r with body %r. '
'Event: %r' % (resp.code, (yield resp.content()), event))
def _forward_event_amqp(self, event):
'''Put the event on the correct queue.'''
if self.config.get('message_queue') is not None:
return self.ro_connector.publish_event(event)
def consume_ack(self, event):
return self.store_and_forward_event(event)
def consume_nack(self, event):
return self.store_and_forward_event(event)
def consume_delivery_report(self, event):
return self.store_and_forward_event(event)
def _get_event_url(self, event):
msg_id = event['user_message_id']
if msg_id is not None:
return self.outbounds.load_event_url(self.channel_id, msg_id)
else:
logging.warning(
"Cannot find event URL, missing user_message_id: %r" % event)
def create_store(self, **kw):
'''
Creates and returns a new message rate store.
'''
redis = yield self.get_redis()
returnValue(MessageRateStore(redis, **kw))
class JunebugApi(object):
app = Klein()
def __init__(self, service, config):
self.service = service
self.redis_config = config.redis
self.amqp_config = config.amqp
self.config = config
@inlineCallbacks
def setup(self, redis=None, message_sender=None):
if redis is None:
redis = yield TxRedisManager.from_config(self.redis_config)
if message_sender is None:
message_sender = MessageSender('amqp-spec-0-8.xml',
self.amqp_config)
self.redis = redis
self.message_sender = message_sender
self.message_sender.setServiceParent(self.service)
self.inbounds = InboundMessageStore(self.redis,
self.config.inbound_message_ttl)
self.outbounds = OutboundMessageStore(self.redis,
self.config.outbound_message_ttl)
self.message_rate = MessageRateStore(self.redis)
self.plugins = []
for plugin_config in self.config.plugins:
cls = load_class_by_string(plugin_config['type'])
plugin = cls()
yield plugin.start_plugin(plugin_config, self.config)
self.plugins.append(plugin)
yield Channel.start_all_channels(self.redis, self.config, self.service,
self.plugins)
@inlineCallbacks
def teardown(self):
yield self.redis.close_manager()
for plugin in self.plugins:
yield plugin.stop_plugin()
@app.handle_errors(JunebugError)
def generic_junebug_error(self, request, failure):
return response(request,
failure.value.description, {
'errors': [{
'type': failure.value.name,
'message': failure.getErrorMessage(),
}]
},
code=failure.value.code)
@app.handle_errors(HTTPException)
def http_error(self, request, failure):
error = {
'code': failure.value.code,
'type': failure.value.name,
'message': failure.getErrorMessage(),
}
if getattr(failure.value, 'new_url', None) is not None:
request.setHeader('Location', failure.value.new_url)
error['new_url'] = failure.value.new_url
return response(request,
failure.value.description, {
'errors': [error],
},
code=failure.value.code)
@app.handle_errors
def generic_error(self, request, failure):
log.err(failure)
return response(request,
'generic error', {
'errors': [{
'type': failure.type.__name__,
'message': failure.getErrorMessage(),
}]
},
code=http.INTERNAL_SERVER_ERROR)
@app.route('/channels/', methods=['GET'])
@inlineCallbacks
def get_channel_list(self, request):
'''List all channels'''
ids = yield Channel.get_all(self.redis)
returnValue(response(request, 'channels listed', sorted(ids)))
@app.route('/channels/', methods=['POST'])
@json_body
@validate(
body_schema({
'type': 'object',
'properties': {
'type': {
'type': 'string'
},
'label': {
'type': 'string'
},
'config': {
'type': 'object'
},
'metadata': {
'type': 'object'
},
'status_url': {
'type': 'string'
},
'mo_url': {
'type': 'string'
},
'mo_url_auth_token': {
'type': 'string'
},
'amqp_queue': {
'type': 'string'
},
'rate_limit_count': {
'type': 'integer',
'minimum': 0,
},
'rate_limit_window': {
'type': 'integer',
'minimum': 0,
},
'character_limit': {
'type': 'integer',
'minimum': 0,
},
},
'required': ['type', 'config'],
}))
@inlineCallbacks
def create_channel(self, request, body):
'''Create a channel'''
if not (body.get('mo_url') or body.get('amqp_queue')):
raise ApiUsageError(
'One or both of "mo_url" and "amqp_queue" must be specified')
channel = Channel(self.redis, self.config, body, self.plugins)
yield channel.start(self.service)
yield channel.save()
returnValue(
response(request, 'channel created', (yield channel.status())))
@app.route('/channels/<string:channel_id>', methods=['GET'])
@inlineCallbacks
def get_channel(self, request, channel_id):
'''Return the channel configuration and a nested status object'''
channel = yield Channel.from_id(self.redis, self.config, channel_id,
self.service, self.plugins)
resp = yield channel.status()
returnValue(response(request, 'channel found', resp))
@app.route('/channels/<string:channel_id>', methods=['POST'])
@json_body
@validate(
body_schema({
'type': 'object',
'properties': {
'type': {
'type': 'string'
},
'label': {
'type': 'string'
},
'config': {
'type': 'object'
},
'metadata': {
'type': 'object'
},
'status_url': {
'type': ['string', 'null']
},
'mo_url': {
'type': ['string', 'null']
},
'rate_limit_count': {
'type': 'integer',
'minimum': 0,
},
'rate_limit_window': {
'type': 'integer',
'minimum': 0,
},
'character_limit': {
'type': 'integer',
'minimum': 0,
},
},
}))
@inlineCallbacks
def modify_channel(self, request, body, channel_id):
'''Mondify the channel configuration'''
channel = yield Channel.from_id(self.redis, self.config, channel_id,
self.service, self.plugins)
resp = yield channel.update(body)
returnValue(response(request, 'channel updated', resp))
@app.route('/channels/<string:channel_id>', methods=['DELETE'])
@inlineCallbacks
def delete_channel(self, request, channel_id):
'''Delete the channel'''
channel = yield Channel.from_id(self.redis, self.config, channel_id,
self.service, self.plugins)
yield channel.stop()
yield channel.delete()
returnValue(response(request, 'channel deleted', {}))
@app.route('/channels/<string:channel_id>/restart', methods=['POST'])
@inlineCallbacks
def restart_channel(self, request, channel_id):
'''Restart a channel.'''
channel = yield Channel.from_id(self.redis, self.config, channel_id,
self.service, self.plugins)
yield channel.stop()
yield channel.start(self.service)
returnValue(response(request, 'channel restarted', {}))
@app.route('/channels/<string:channel_id>/logs', methods=['GET'])
@inlineCallbacks
def get_logs(self, request, channel_id):
'''Get the last N logs for a channel, sorted reverse
chronologically.'''
n = request.args.get('n', None)
if n is not None:
n = int(n[0])
channel = yield Channel.from_id(self.redis, self.config, channel_id,
self.service, self.plugins)
logs = yield channel.get_logs(n)
returnValue(response(request, 'logs retrieved', logs))
@app.route('/channels/<string:channel_id>/messages/', methods=['POST'])
@json_body
@validate(
body_schema({
'type': 'object',
'properties': {
'to': {
'type': 'string'
},
'from': {
'type': ['string', 'null']
},
'reply_to': {
'type': 'string'
},
'content': {
'type': ['string', 'null']
},
'event_url': {
'type': 'string'
},
'priority': {
'type': 'string'
},
'channel_data': {
'type': 'object'
},
},
'required': ['content'],
'additionalProperties': False,
}))
@inlineCallbacks
def send_message(self, request, body, channel_id):
'''Send an outbound (mobile terminated) message'''
if 'to' not in body and 'reply_to' not in body:
raise ApiUsageError('Either "to" or "reply_to" must be specified')
if 'to' in body and 'reply_to' in body:
raise ApiUsageError(
'Only one of "to" and "reply_to" may be specified')
if 'from' in body and 'reply_to' in body:
raise ApiUsageError(
'Only one of "from" and "reply_to" may be specified')
channel = yield Channel.from_id(self.redis, self.config, channel_id,
self.service, self.plugins)
if 'to' in body:
msg = yield channel.send_message(self.message_sender,
self.outbounds, body)
else:
msg = yield channel.send_reply_message(self.message_sender,
self.outbounds,
self.inbounds, body)
yield self.message_rate.increment(channel_id, 'outbound',
self.config.metric_window)
returnValue(response(request, 'message sent', msg))
@app.route('/channels/<string:channel_id>/messages/<string:message_id>',
methods=['GET'])
@inlineCallbacks
def get_message_status(self, request, channel_id, message_id):
'''Retrieve the status of a message'''
events = yield self.outbounds.load_all_events(channel_id, message_id)
events = sorted((api_from_event(channel_id, e) for e in events),
key=lambda e: e['timestamp'])
last_event = events[-1] if events else None
last_event_type = last_event['event_type'] if last_event else None
last_event_timestamp = last_event['timestamp'] if last_event else None
returnValue(
response(
request, 'message status', {
'id': message_id,
'last_event_type': last_event_type,
'last_event_timestamp': last_event_timestamp,
'events': events,
}))
@app.route('/health', methods=['GET'])
def health_status(self, request):
return response(request, 'health ok', {})
class JunebugApi(object):
app = Klein()
def __init__(self, service, config):
self.service = service
self.redis_config = config.redis
self.amqp_config = config.amqp
self.config = config
@inlineCallbacks
def setup(self, redis=None, message_sender=None):
if redis is None:
redis = yield TxRedisManager.from_config(self.redis_config)
if message_sender is None:
message_sender = MessageSender(
'amqp-spec-0-8.xml', self.amqp_config)
self.redis = redis
self.message_sender = message_sender
self.message_sender.setServiceParent(self.service)
self.inbounds = InboundMessageStore(
self.redis, self.config.inbound_message_ttl)
self.outbounds = OutboundMessageStore(
self.redis, self.config.outbound_message_ttl)
self.message_rate = MessageRateStore(self.redis)
self.plugins = []
for plugin_config in self.config.plugins:
cls = load_class_by_string(plugin_config['type'])
plugin = cls()
yield plugin.start_plugin(plugin_config, self.config)
self.plugins.append(plugin)
yield Channel.start_all_channels(
self.redis, self.config, self.service, self.plugins)
@inlineCallbacks
def teardown(self):
yield self.redis.close_manager()
for plugin in self.plugins:
yield plugin.stop_plugin()
@app.handle_errors(JunebugError)
def generic_junebug_error(self, request, failure):
return response(request, failure.value.description, {
'errors': [{
'type': failure.value.name,
'message': failure.getErrorMessage(),
}]
}, code=failure.value.code)
@app.handle_errors(HTTPException)
def http_error(self, request, failure):
return response(request, failure.value.description, {
'errors': [{
'type': failure.value.name,
'message': failure.getErrorMessage(),
}]
}, code=failure.value.code)
@app.handle_errors
def generic_error(self, request, failure):
logging.exception(failure)
return response(request, 'generic error', {
'errors': [{
'type': failure.type.__name__,
'message': failure.getErrorMessage(),
}]
}, code=http.INTERNAL_SERVER_ERROR)
@app.route('/channels/', methods=['GET'])
@inlineCallbacks
def get_channel_list(self, request):
'''List all channels'''
ids = yield Channel.get_all(self.redis)
returnValue(response(request, 'channels listed', sorted(ids)))
@app.route('/channels/', methods=['POST'])
@json_body
@validate(
body_schema({
'type': 'object',
'properties': {
'type': {'type': 'string'},
'label': {'type': 'string'},
'config': {'type': 'object'},
'metadata': {'type': 'object'},
'status_url': {'type': 'string'},
'mo_url': {'type': 'string'},
'amqp_queue': {'type': 'string'},
'rate_limit_count': {
'type': 'integer',
'minimum': 0,
},
'rate_limit_window': {
'type': 'integer',
'minimum': 0,
},
'character_limit': {
'type': 'integer',
'minimum': 0,
},
},
'required': ['type', 'config'],
}))
@inlineCallbacks
def create_channel(self, request, body):
'''Create a channel'''
if not (body.get('mo_url') or body.get('amqp_queue')):
raise ApiUsageError(
'One or both of "mo_url" and "amqp_queue" must be specified')
channel = Channel(
self.redis, self.config, body, self.plugins)
yield channel.save()
yield channel.start(self.service)
returnValue(response(
request, 'channel created', (yield channel.status())))
@app.route('/channels/<string:channel_id>', methods=['GET'])
@inlineCallbacks
def get_channel(self, request, channel_id):
'''Return the channel configuration and a nested status object'''
channel = yield Channel.from_id(
self.redis, self.config, channel_id, self.service, self.plugins)
resp = yield channel.status()
returnValue(response(
request, 'channel found', resp))
@app.route('/channels/<string:channel_id>', methods=['POST'])
@json_body
@validate(
body_schema({
'type': 'object',
'properties': {
'type': {'type': 'string'},
'label': {'type': 'string'},
'config': {'type': 'object'},
'metadata': {'type': 'object'},
'status_url': {'type': 'string'},
'mo_url': {'type': 'string'},
'rate_limit_count': {
'type': 'integer',
'minimum': 0,
},
'rate_limit_window': {
'type': 'integer',
'minimum': 0,
},
'character_limit': {
'type': 'integer',
'minimum': 0,
},
},
}))
@inlineCallbacks
def modify_channel(self, request, body, channel_id):
'''Mondify the channel configuration'''
channel = yield Channel.from_id(
self.redis, self.config, channel_id, self.service, self.plugins)
resp = yield channel.update(body)
returnValue(response(
request, 'channel updated', resp))
@app.route('/channels/<string:channel_id>', methods=['DELETE'])
@inlineCallbacks
def delete_channel(self, request, channel_id):
'''Delete the channel'''
channel = yield Channel.from_id(
self.redis, self.config, channel_id, self.service, self.plugins)
yield channel.stop()
yield channel.delete()
returnValue(response(
request, 'channel deleted', {}))
@app.route('/channels/<string:channel_id>/logs', methods=['GET'])
@inlineCallbacks
def get_logs(self, request, channel_id):
'''Get the last N logs for a channel, sorted reverse
chronologically.'''
n = request.args.get('n', None)
if n is not None:
n = int(n[0])
channel = yield Channel.from_id(
self.redis, self.config, channel_id, self.service, self.plugins)
logs = yield channel.get_logs(n)
returnValue(response(request, 'logs retrieved', logs))
@app.route('/channels/<string:channel_id>/messages/', methods=['POST'])
@json_body
@validate(
body_schema({
'type': 'object',
'properties': {
'to': {'type': 'string'},
'from': {'type': ['string', 'null']},
'reply_to': {'type': 'string'},
'content': {'type': ['string', 'null']},
'event_url': {'type': 'string'},
'priority': {'type': 'string'},
'channel_data': {'type': 'object'},
},
'required': ['content'],
'additionalProperties': False,
}))
@inlineCallbacks
def send_message(self, request, body, channel_id):
'''Send an outbound (mobile terminated) message'''
if 'to' not in body and 'reply_to' not in body:
raise ApiUsageError(
'Either "to" or "reply_to" must be specified')
if 'to' in body and 'reply_to' in body:
raise ApiUsageError(
'Only one of "to" and "reply_to" may be specified')
if 'from' in body and 'reply_to' in body:
raise ApiUsageError(
'Only one of "from" and "reply_to" may be specified')
channel = yield Channel.from_id(
self.redis, self.config, channel_id, self.service, self.plugins)
if 'to' in body:
msg = yield channel.send_message(
self.message_sender, self.outbounds, body)
else:
msg = yield channel.send_reply_message(
self.message_sender, self.outbounds, self.inbounds, body)
yield self.message_rate.increment(
channel_id, 'outbound', self.config.metric_window)
returnValue(response(request, 'message sent', msg))
@app.route(
'/channels/<string:channel_id>/messages/<string:message_id>',
methods=['GET'])
@inlineCallbacks
def get_message_status(self, request, channel_id, message_id):
'''Retrieve the status of a message'''
events = yield self.outbounds.load_all_events(channel_id, message_id)
events = sorted(
(api_from_event(channel_id, e) for e in events),
key=lambda e: e['timestamp'])
last_event = events[-1] if events else None
last_event_type = last_event['event_type'] if last_event else None
last_event_timestamp = last_event['timestamp'] if last_event else None
returnValue(response(request, 'message status', {
'id': message_id,
'last_event_type': last_event_type,
'last_event_timestamp': last_event_timestamp,
'events': events,
}))
@app.route('/health', methods=['GET'])
def health_status(self, request):
return response(request, 'health ok', {})
class MessageForwardingWorker(ApplicationWorker):
'''This application worker consumes vumi messages placed on a configured
amqp queue, and sends them as HTTP requests with a JSON body to a
configured URL'''
CONFIG_CLASS = MessageForwardingConfig
@inlineCallbacks
def setup_application(self):
self.redis = yield TxRedisManager.from_config(
self.config['redis_manager'])
self.inbounds = InboundMessageStore(
self.redis, self.config['inbound_ttl'])
self.outbounds = OutboundMessageStore(
self.redis, self.config['outbound_ttl'])
self.message_rate = MessageRateStore(self.redis)
if self.config.get('message_queue') is not None:
self.ro_connector = yield self.setup_ro_connector(
self.config['message_queue'])
self.ro_connector.set_outbound_handler(
self._publish_message)
@inlineCallbacks
def teardown_application(self):
yield self.redis.close_manager()
@property
def channel_id(self):
return self.config['transport_name']
@inlineCallbacks
def consume_user_message(self, message):
'''Sends the vumi message as an HTTP request to the configured URL'''
yield self.inbounds.store_vumi_message(self.channel_id, message)
msg = api_from_message(message)
if self.config.get('mo_message_url') is not None:
resp = yield post(self.config['mo_message_url'], msg)
if request_failed(resp):
logging.exception(
'Error sending message, received HTTP code %r with body %r'
'. Message: %r' % (resp.code, (yield resp.content()), msg))
if self.config.get('message_queue') is not None:
yield self.ro_connector.publish_inbound(message)
yield self._increment_metric('inbound')
@inlineCallbacks
def store_and_forward_event(self, event):
'''Store the event in the message store, POST it to the correct
URL.'''
yield self._store_event(event)
yield self._forward_event(event)
yield self._count_event(event)
def _increment_metric(self, label):
return self.message_rate.increment(
self.channel_id, label, self.config['metric_window'])
def _count_event(self, event):
if event['event_type'] == 'ack':
return self._increment_metric('submitted')
if event['event_type'] == 'nack':
return self._increment_metric('rejected')
if event['event_type'] == 'delivery_report':
if event['delivery_status'] == 'pending':
return self._increment_metric('delivery_pending')
if event['delivery_status'] == 'failed':
return self._increment_metric('delivery_failed')
if event['delivery_status'] == 'delivered':
return self._increment_metric('delivery_succeeded')
def _store_event(self, event):
'''Stores the event in the message store'''
message_id = event['user_message_id']
return self.outbounds.store_event(self.channel_id, message_id, event)
@inlineCallbacks
def _forward_event(self, event):
'''Forward the event to the correct places.'''
yield self._forward_event_http(event)
yield self._forward_event_amqp(event)
@inlineCallbacks
def _forward_event_http(self, event):
'''POST the event to the correct URL'''
url = yield self._get_event_url(event)
if url is None:
return
msg = api_from_event(self.channel_id, event)
if msg['event_type'] is None:
logging.exception("Discarding unrecognised event %r" % (event,))
return
resp = yield post(url, msg)
if request_failed(resp):
logging.exception(
'Error sending event, received HTTP code %r with body %r. '
'Event: %r' % (resp.code, (yield resp.content()), event))
def _forward_event_amqp(self, event):
'''Put the event on the correct queue.'''
if self.config.get('message_queue') is not None:
return self.ro_connector.publish_event(event)
def consume_ack(self, event):
return self.store_and_forward_event(event)
def consume_nack(self, event):
return self.store_and_forward_event(event)
def consume_delivery_report(self, event):
return self.store_and_forward_event(event)
def _get_event_url(self, event):
msg_id = event['user_message_id']
return self.outbounds.load_event_url(self.channel_id, msg_id)
