def setup(self, redis=None, message_sender=None): if redis is None: redis = yield TxRedisManager.from_config(self.redis_config) if message_sender is None: message_sender = MessageSender('amqp-spec-0-8.xml', self.amqp_config) self.redis = redis self.message_sender = message_sender self.message_sender.setServiceParent(self.service) self.inbounds = InboundMessageStore(self.redis, self.config.inbound_message_ttl) self.outbounds = OutboundMessageStore(self.redis, self.config.outbound_message_ttl) self.message_rate = MessageRateStore(self.redis) self.plugins = [] for plugin_config in self.config.plugins: cls = load_class_by_string(plugin_config['type']) plugin = cls() yield plugin.start_plugin(plugin_config, self.config) self.plugins.append(plugin) yield Channel.start_all_channels(self.redis, self.config, self.service, self.plugins)
def setup_application(self): self.redis = yield TxRedisManager.from_config( self.config['redis_manager']) self.inbounds = InboundMessageStore(self.redis, self.config['inbound_ttl']) self.outbounds = OutboundMessageStore(self.redis, self.config['outbound_ttl']) self.message_rate = MessageRateStore(self.redis) if self.config.get('message_queue') is not None: self.ro_connector = yield self.setup_ro_connector( self.config['message_queue']) self.ro_connector.set_outbound_handler(self._publish_message)
def setup(self, redis=None, message_sender=None): if redis is None: redis = yield TxRedisManager.from_config(self.redis_config) if message_sender is None: message_sender = MessageSender( 'amqp-spec-0-8.xml', self.amqp_config) self.redis = redis self.message_sender = message_sender self.message_sender.setServiceParent(self.service) self.inbounds = InboundMessageStore( self.redis, self.config.inbound_message_ttl) self.outbounds = OutboundMessageStore( self.redis, self.config.outbound_message_ttl) self.message_rate = MessageRateStore(self.redis) self.plugins = [] for plugin_config in self.config.plugins: cls = load_class_by_string(plugin_config['type']) plugin = cls() yield plugin.start_plugin(plugin_config, self.config) self.plugins.append(plugin) yield Channel.start_all_channels( self.redis, self.config, self.service, self.plugins)
def __init__(self, redis_manager, config, properties, plugins=[], id=None): '''Creates a new channel. ``redis_manager`` is the redis manager, from which a sub manager is created using the channel id. If the channel id is not supplied, a UUID one is generated. Call ``save`` to save the channel data. It can be started using the ``start`` function.''' self._properties = properties self.redis = redis_manager self.id = id self.config = config if self.id is None: self.id = str(uuid.uuid4()) self.options = deepcopy(VumiOptions.default_vumi_options) self.options.update(self.config.amqp) self.transport_worker = None self.application_worker = None self.status_application_worker = None self.sstore = StatusStore(self.redis) self.plugins = plugins self.message_rates = MessageRateStore(self.redis)
def setup_application(self): self.redis = yield TxRedisManager.from_config( self.config['redis_manager']) self.inbounds = InboundMessageStore( self.redis, self.config['inbound_ttl']) self.outbounds = OutboundMessageStore( self.redis, self.config['outbound_ttl']) self.message_rate = MessageRateStore(self.redis) if self.config.get('message_queue') is not None: self.ro_connector = yield self.setup_ro_connector( self.config['message_queue']) self.ro_connector.set_outbound_handler( self._publish_message)
class MessageForwardingWorker(ApplicationWorker): '''This application worker consumes vumi messages placed on a configured amqp queue, and sends them as HTTP requests with a JSON body to a configured URL''' CONFIG_CLASS = MessageForwardingConfig @inlineCallbacks def setup_application(self): self.redis = yield TxRedisManager.from_config( self.config['redis_manager']) self.inbounds = InboundMessageStore(self.redis, self.config['inbound_ttl']) self.outbounds = OutboundMessageStore(self.redis, self.config['outbound_ttl']) self.message_rate = MessageRateStore(self.redis) if self.config.get('message_queue') is not None: self.ro_connector = yield self.setup_ro_connector( self.config['message_queue']) self.ro_connector.set_outbound_handler(self._publish_message) @inlineCallbacks def teardown_application(self): yield self.redis.close_manager() @property def channel_id(self): return self.config['transport_name'] @inlineCallbacks def consume_user_message(self, message): '''Sends the vumi message as an HTTP request to the configured URL''' yield self.inbounds.store_vumi_message(self.channel_id, message) msg = api_from_message(message) if self.config.get('mo_message_url') is not None: resp = yield post(self.config['mo_message_url'], msg) if request_failed(resp): logging.exception( 'Error sending message, received HTTP code %r with body %r' '. Message: %r' % (resp.code, (yield resp.content()), msg)) if self.config.get('message_queue') is not None: yield self.ro_connector.publish_inbound(message) yield self._increment_metric('inbound') @inlineCallbacks def store_and_forward_event(self, event): '''Store the event in the message store, POST it to the correct URL.''' yield self._store_event(event) yield self._forward_event(event) yield self._count_event(event) def _increment_metric(self, label): return self.message_rate.increment(self.channel_id, label, self.config['metric_window']) def _count_event(self, event): if event['event_type'] == 'ack': return self._increment_metric('submitted') if event['event_type'] == 'nack': return self._increment_metric('rejected') if event['event_type'] == 'delivery_report': if event['delivery_status'] == 'pending': return self._increment_metric('delivery_pending') if event['delivery_status'] == 'failed': return self._increment_metric('delivery_failed') if event['delivery_status'] == 'delivered': return self._increment_metric('delivery_succeeded') def _store_event(self, event): '''Stores the event in the message store''' message_id = event['user_message_id'] return self.outbounds.store_event(self.channel_id, message_id, event) @inlineCallbacks def _forward_event(self, event): '''Forward the event to the correct places.''' yield self._forward_event_http(event) yield self._forward_event_amqp(event) @inlineCallbacks def _forward_event_http(self, event): '''POST the event to the correct URL''' url = yield self._get_event_url(event) if url is None: return msg = api_from_event(self.channel_id, event) if msg['event_type'] is None: logging.exception("Discarding unrecognised event %r" % (event, )) return resp = yield post(url, msg) if request_failed(resp): logging.exception( 'Error sending event, received HTTP code %r with body %r. ' 'Event: %r' % (resp.code, (yield resp.content()), event)) def _forward_event_amqp(self, event): '''Put the event on the correct queue.''' if self.config.get('message_queue') is not None: return self.ro_connector.publish_event(event) def consume_ack(self, event): return self.store_and_forward_event(event) def consume_nack(self, event): return self.store_and_forward_event(event) def consume_delivery_report(self, event): return self.store_and_forward_event(event) def _get_event_url(self, event): msg_id = event['user_message_id'] return self.outbounds.load_event_url(self.channel_id, msg_id)
class Channel(object): OUTBOUND_QUEUE = '%s.outbound' APPLICATION_ID = 'application:%s' STATUS_APPLICATION_ID = 'status:%s' APPLICATION_CLS_NAME = 'junebug.workers.MessageForwardingWorker' STATUS_APPLICATION_CLS_NAME = 'junebug.workers.ChannelStatusWorker' JUNEBUG_LOGGING_SERVICE_CLS = JunebugLoggerService def __init__(self, redis_manager, config, properties, plugins=[], id=None): '''Creates a new channel. ``redis_manager`` is the redis manager, from which a sub manager is created using the channel id. If the channel id is not supplied, a UUID one is generated. Call ``save`` to save the channel data. It can be started using the ``start`` function.''' self._properties = properties self.redis = redis_manager self.id = id self.config = config if self.id is None: self.id = str(uuid.uuid4()) self.options = deepcopy(VumiOptions.default_vumi_options) self.options.update(self.config.amqp) self.transport_worker = None self.application_worker = None self.status_application_worker = None self.sstore = StatusStore(self.redis) self.plugins = plugins self.message_rates = MessageRateStore(self.redis) @property def application_id(self): return self.APPLICATION_ID % (self.id,) @property def status_application_id(self): return self.STATUS_APPLICATION_ID % (self.id,) @property def character_limit(self): return self._properties.get('character_limit') @inlineCallbacks def start(self, service, transport_worker=None): '''Starts the relevant workers for the channel. ``service`` is the parent of under which the workers should be started.''' self._start_transport(service, transport_worker) self._start_application(service) self._start_status_application(service) for plugin in self.plugins: yield plugin.channel_started(self) @inlineCallbacks def stop(self): '''Stops the relevant workers for the channel''' yield self._stop_application() yield self._stop_status_application() yield self._stop_transport() for plugin in self.plugins: yield plugin.channel_stopped(self) @inlineCallbacks def save(self): '''Saves the channel data into redis.''' properties = json.dumps(self._properties) channel_redis = yield self.redis.sub_manager(self.id) yield channel_redis.set('properties', properties) yield self.redis.sadd('channels', self.id) @inlineCallbacks def update(self, properties): '''Updates the channel configuration, saves the updated configuration, and (if needed) restarts the channel with the new configuration. Returns the updated configuration and status.''' self._properties.update(properties) yield self.save() service = self.transport_worker.parent # Only restart if the channel config has changed if 'config' in properties: yield self._stop_transport() yield self._start_transport(service) if 'mo_url' in properties: yield self._stop_application() yield self._start_application(service) returnValue((yield self.status())) @inlineCallbacks def delete(self): '''Removes the channel data from redis''' channel_redis = yield self.redis.sub_manager(self.id) yield channel_redis.delete('properties') yield self.redis.srem('channels', self.id) @inlineCallbacks def status(self): '''Returns a dict with the configuration and status of the channel''' status = deepcopy(self._properties) status['id'] = self.id status['status'] = yield self._get_status() returnValue(status) def _get_message_rate(self, label): return self.message_rates.get_messages_per_second( self.id, label, self.config.metric_window) @inlineCallbacks def _get_status(self): components = yield self.sstore.get_statuses(self.id) components = dict( (k, api_from_status(self.id, v)) for k, v in components.iteritems() ) status_values = { 'down': 0, 'degraded': 1, 'ok': 2, } try: status = min( (c['status'] for c in components.values()), key=status_values.get) except ValueError: # No statuses status = None returnValue({ 'components': components, 'status': status, 'inbound_message_rate': ( yield self._get_message_rate('inbound')), 'outbound_message_rate': ( yield self._get_message_rate('outbound')), 'submitted_event_rate': ( yield self._get_message_rate('submitted')), 'rejected_event_rate': ( yield self._get_message_rate('rejected')), 'delivery_succeeded_rate': ( yield self._get_message_rate('delivery_succeeded')), 'delivery_failed_rate': ( yield self._get_message_rate('delivery_failed')), 'delivery_pending_rate': ( yield self._get_message_rate('delivery_pending')), }) @classmethod @inlineCallbacks def from_id(cls, redis, config, id, parent, plugins=[]): '''Creates a channel by loading the data from redis, given the channel's id, and the parent service of the channel''' channel_redis = yield redis.sub_manager(id) properties = yield channel_redis.get('properties') if properties is None: raise ChannelNotFound() properties = json.loads(properties) obj = cls(redis, config, properties, plugins, id=id) obj._restore(parent) returnValue(obj) @classmethod @inlineCallbacks def get_all(cls, redis): '''Returns a set of keys of all of the channels''' channels = yield redis.smembers('channels') returnValue(channels) @classmethod @inlineCallbacks def start_all_channels(cls, redis, config, parent, plugins=[]): '''Ensures that all of the stored channels are running''' for id in (yield cls.get_all(redis)): if id not in parent.namedServices: properties = json.loads(( yield redis.get('%s:properties' % id))) channel = cls(redis, config, properties, plugins, id=id) yield channel.start(parent) @inlineCallbacks def send_message(self, sender, outbounds, msg): '''Sends a message.''' event_url = msg.get('event_url') msg = message_from_api(self.id, msg) msg = TransportUserMessage.send(**msg) msg = yield self._send_message(sender, outbounds, event_url, msg) returnValue(api_from_message(msg)) @inlineCallbacks def send_reply_message(self, sender, outbounds, inbounds, msg): '''Sends a reply message.''' in_msg = yield inbounds.load_vumi_message(self.id, msg['reply_to']) if in_msg is None: raise MessageNotFound( "Inbound message with id %s not found" % (msg['reply_to'],)) event_url = msg.get('event_url') msg = message_from_api(self.id, msg) msg = in_msg.reply(**msg) msg = yield self._send_message(sender, outbounds, event_url, msg) returnValue(api_from_message(msg)) def get_logs(self, n): '''Returns the last `n` logs. If `n` is greater than the configured limit, only returns the configured limit amount of logs. If `n` is None, returns the configured limit amount of logs.''' if n is None: n = self.config.max_logs n = min(n, self.config.max_logs) logfile = self.transport_worker.getServiceNamed( 'Junebug Worker Logger').logfile return read_logs(logfile, n) @property def _transport_config(self): config = self._properties['config'] config = self._convert_unicode(config) config['transport_name'] = self.id config['worker_name'] = self.id config['publish_status'] = True return config @property def _application_config(self): return { 'transport_name': self.id, 'mo_message_url': self._properties['mo_url'], 'redis_manager': self.config.redis, 'inbound_ttl': self.config.inbound_message_ttl, 'outbound_ttl': self.config.outbound_message_ttl, 'metric_window': self.config.metric_window, } @property def _status_application_config(self): return { 'redis_manager': self.config.redis, 'channel_id': self.id, 'status_url': self._properties.get('status_url'), } @property def _available_transports(self): if self.config.replace_channels: return self.config.channels else: channels = {} channels.update(transports) channels.update(self.config.channels) return channels @property def _transport_cls_name(self): cls_name = self._available_transports.get(self._properties.get('type')) if cls_name is None: raise InvalidChannelType( 'Invalid channel type %r, must be one of: %s' % ( self._properties.get('type'), ', '.join(self._available_transports.keys()))) return cls_name def _start_transport(self, service, transport_worker=None): # transport_worker parameter is for testing, if it is None, # create the transport worker if transport_worker is None: transport_worker = self._create_transport() transport_worker.setName(self.id) logging_service = self._create_junebug_logger_service() transport_worker.addService(logging_service) transport_worker.setServiceParent(service) self.transport_worker = transport_worker def _start_application(self, service): worker = self._create_application() worker.setName(self.application_id) worker.setServiceParent(service) self.application_worker = worker def _start_status_application(self, service): worker = self._create_status_application() worker.setName(self.status_application_id) worker.setServiceParent(service) self.status_application_worker = worker def _create_transport(self): return self._create_worker( self._transport_cls_name, self._transport_config) def _create_application(self): return self._create_worker( self.APPLICATION_CLS_NAME, self._application_config) def _create_status_application(self): return self._create_worker( self.STATUS_APPLICATION_CLS_NAME, self._status_application_config) def _create_junebug_logger_service(self): return self.JUNEBUG_LOGGING_SERVICE_CLS( self.id, self.config.logging_path, self.config.log_rotate_size, self.config.max_log_files) def _create_worker(self, cls_name, config): creator = WorkerCreator(self.options) worker = creator.create_worker(cls_name, config) return worker @inlineCallbacks def _stop_transport(self): if self.transport_worker is not None: yield self.transport_worker.disownServiceParent() self.transport_worker = None @inlineCallbacks def _stop_application(self): if self.application_worker is not None: yield self.application_worker.disownServiceParent() self.application_worker = None @inlineCallbacks def _stop_status_application(self): if self.status_application_worker is not None: yield self.status_application_worker.disownServiceParent() self.status_application_worker = None def _restore(self, service): self.transport_worker = service.getServiceNamed(self.id) self.application_worker = service.getServiceNamed(self.application_id) self.status_application_worker = service.getServiceNamed( self.status_application_id) def _convert_unicode(self, data): # Twisted doesn't like it when we give unicode in for config things if isinstance(data, basestring): return str(data) elif isinstance(data, collections.Mapping): return dict(map(self._convert_unicode, data.iteritems())) elif isinstance(data, collections.Iterable): return type(data)(map(self._convert_unicode, data)) else: return data def _check_character_limit(self, content): count = len(content) if (self.character_limit is not None and count > self.character_limit): raise MessageTooLong( 'Message content %r is of length %d, which is greater than the' ' character limit of %d' % ( content, count, self.character_limit) ) @inlineCallbacks def _send_message(self, sender, outbounds, event_url, msg): self._check_character_limit(msg['content']) if event_url is not None: yield outbounds.store_event_url( self.id, msg['message_id'], event_url) queue = self.OUTBOUND_QUEUE % (self.id,) msg = yield sender.send_message(msg, routing_key=queue) returnValue(msg)
class MessageForwardingWorker(ApplicationWorker): '''This application worker consumes vumi messages placed on a configured amqp queue, and sends them as HTTP requests with a JSON body to a configured URL''' CONFIG_CLASS = MessageForwardingConfig @inlineCallbacks def setup_application(self): self.redis = yield TxRedisManager.from_config( self.config['redis_manager']) self.inbounds = InboundMessageStore( self.redis, self.config['inbound_ttl']) self.outbounds = OutboundMessageStore( self.redis, self.config['outbound_ttl']) self.message_rate = MessageRateStore(self.redis) if self.config.get('message_queue') is not None: self.ro_connector = yield self.setup_ro_connector( self.config['message_queue']) self.ro_connector.set_outbound_handler( self._publish_message) @inlineCallbacks def teardown_application(self): if getattr(self, 'redis', None) is not None: yield self.redis.close_manager() @property def channel_id(self): return self.config['transport_name'] @inlineCallbacks def consume_user_message(self, message): '''Sends the vumi message as an HTTP request to the configured URL''' yield self.inbounds.store_vumi_message(self.channel_id, message) msg = api_from_message(message) if self.config.get('mo_message_url') is not None: config = self.get_static_config() (url, auth) = self._split_url_and_credentials( config.mo_message_url) # Construct token auth headers if configured auth_token = config.mo_message_url_auth_token if auth_token: headers = { 'Authorization': [ 'Token %s' % (auth_token,)] } else: headers = {} resp = yield post(url, msg, timeout=config.mo_message_url_timeout, auth=auth, headers=headers) if resp and request_failed(resp): logging.exception( 'Error sending message, received HTTP code %r with body %r' '. Message: %r' % (resp.code, (yield resp.content()), msg)) if self.config.get('message_queue') is not None: yield self.ro_connector.publish_inbound(message) yield self._increment_metric('inbound') @inlineCallbacks def store_and_forward_event(self, event): '''Store the event in the message store, POST it to the correct URL.''' yield self._store_event(event) yield self._forward_event(event) yield self._count_event(event) def _increment_metric(self, label): return self.message_rate.increment( self.channel_id, label, self.config['metric_window']) def _count_event(self, event): if event['event_type'] == 'ack': return self._increment_metric('submitted') if event['event_type'] == 'nack': return self._increment_metric('rejected') if event['event_type'] == 'delivery_report': if event['delivery_status'] == 'pending': return self._increment_metric('delivery_pending') if event['delivery_status'] == 'failed': return self._increment_metric('delivery_failed') if event['delivery_status'] == 'delivered': return self._increment_metric('delivery_succeeded') def _store_event(self, event): '''Stores the event in the message store''' message_id = event['user_message_id'] if message_id is None: logging.warning( "Cannot store event, missing user_message_id: %r" % event) else: return self.outbounds.store_event( self.channel_id, message_id, event) @inlineCallbacks def _forward_event(self, event): '''Forward the event to the correct places.''' yield self._forward_event_http(event) yield self._forward_event_amqp(event) @inlineCallbacks def _forward_event_http(self, event): '''POST the event to the correct URL''' url = yield self._get_event_url(event) if url is None: return (url, auth) = self._split_url_and_credentials(urlparse(url)) # Construct token auth headers if configured auth_token = yield self._get_event_auth_token(event) if auth_token: headers = { 'Authorization': [ 'Token %s' % (auth_token,)] } else: headers = {} msg = api_from_event(self.channel_id, event) if msg['event_type'] is None: logging.exception("Discarding unrecognised event %r" % (event,)) return config = self.get_static_config() resp = yield post(url, msg, timeout=config.event_url_timeout, auth=auth, headers=headers) if resp and request_failed(resp): logging.exception( 'Error sending event, received HTTP code %r with body %r. ' 'Event: %r' % (resp.code, (yield resp.content()), event)) def _forward_event_amqp(self, event): '''Put the event on the correct queue.''' if self.config.get('message_queue') is not None: return self.ro_connector.publish_event(event) def consume_ack(self, event): return self.store_and_forward_event(event) def consume_nack(self, event): return self.store_and_forward_event(event) def consume_delivery_report(self, event): return self.store_and_forward_event(event) def _split_url_and_credentials(self, url): # Parse the basic auth username & password if available username = url.username password = url.password if any([username, password]): url = urlunparse(( url.scheme, '%s%s' % (url.hostname, (':%s' % (url.port,) if url.port is not None else '')), url.path, url.params, url.query, url.fragment, )) auth = (username, password) return (url, auth) return (url.geturl(), None) def _get_event_url(self, event): msg_id = event['user_message_id'] if msg_id is not None: return self.outbounds.load_event_url(self.channel_id, msg_id) else: logging.warning( "Cannot find event URL, missing user_message_id: %r" % event) def _get_event_auth_token(self, event): msg_id = event['user_message_id'] if msg_id is not None: return self.outbounds.load_event_auth_token( self.channel_id, msg_id) else: logging.warning( "Cannot find event auth, missing user_message_id: %r" % event)
class JunebugApi(object): app = Klein() def __init__(self, service, config): self.service = service self.redis_config = config.redis self.amqp_config = config.amqp self.config = config @inlineCallbacks def setup(self, redis=None, message_sender=None): if redis is None: redis = yield TxRedisManager.from_config(self.redis_config) if message_sender is None: message_sender = MessageSender( 'amqp-spec-0-8.xml', self.amqp_config) self.redis = redis self.message_sender = message_sender self.message_sender.setServiceParent(self.service) self.inbounds = InboundMessageStore( self.redis, self.config.inbound_message_ttl) self.outbounds = OutboundMessageStore( self.redis, self.config.outbound_message_ttl) self.message_rate = MessageRateStore(self.redis) self.router_store = RouterStore(self.redis) self.plugins = [] for plugin_config in self.config.plugins: cls = load_class_by_string(plugin_config['type']) plugin = cls() yield plugin.start_plugin(plugin_config, self.config) self.plugins.append(plugin) yield Channel.start_all_channels( self.redis, self.config, self.service, self.plugins) yield Router.start_all_routers(self) if self.config.rabbitmq_management_interface: self.rabbitmq_management_client = RabbitmqManagementClient( self.config.rabbitmq_management_interface, self.amqp_config['username'], self.amqp_config['password']) @inlineCallbacks def teardown(self): yield self.redis.close_manager() for plugin in self.plugins: yield plugin.stop_plugin() @app.handle_errors(JunebugError) def generic_junebug_error(self, request, failure): return response(request, failure.value.description, { 'errors': [{ 'type': failure.value.name, 'message': failure.getErrorMessage(), }] }, code=failure.value.code) @app.handle_errors(HTTPException) def http_error(self, request, failure): error = { 'code': failure.value.code, 'type': failure.value.name, 'message': failure.getErrorMessage(), } if getattr(failure.value, 'new_url', None) is not None: request.setHeader('Location', failure.value.new_url) error['new_url'] = failure.value.new_url return response(request, failure.value.description, { 'errors': [error], }, code=failure.value.code) @app.handle_errors def generic_error(self, request, failure): log.err(failure) return response(request, 'generic error', { 'errors': [{ 'type': failure.type.__name__, 'message': failure.getErrorMessage(), }] }, code=http.INTERNAL_SERVER_ERROR) @app.route('/channels/', methods=['GET']) @inlineCallbacks def get_channel_list(self, request): '''List all channels''' ids = yield Channel.get_all(self.redis) returnValue(response(request, 'channels listed', sorted(ids))) @app.route('/channels/', methods=['POST']) @json_body @validate( body_schema({ 'type': 'object', 'properties': { 'type': {'type': 'string'}, 'label': {'type': 'string'}, 'config': {'type': 'object'}, 'metadata': {'type': 'object'}, 'status_url': {'type': 'string'}, 'mo_url': {'type': 'string'}, 'mo_url_auth_token': {'type': 'string'}, 'amqp_queue': {'type': 'string'}, 'rate_limit_count': { 'type': 'integer', 'minimum': 0, }, 'rate_limit_window': { 'type': 'integer', 'minimum': 0, }, 'character_limit': { 'type': 'integer', 'minimum': 0, }, }, 'required': ['type', 'config'], })) @inlineCallbacks def create_channel(self, request, body): '''Create a channel''' channel = Channel( self.redis, self.config, body, self.plugins) yield channel.start(self.service) yield channel.save() returnValue(response( request, 'channel created', (yield channel.status()), code=http.CREATED)) @app.route('/channels/<string:channel_id>', methods=['GET']) @inlineCallbacks def get_channel(self, request, channel_id): '''Return the channel configuration and a nested status object''' channel = yield Channel.from_id( self.redis, self.config, channel_id, self.service, self.plugins) resp = yield channel.status() returnValue(response( request, 'channel found', resp)) @app.route('/channels/<string:channel_id>', methods=['POST']) @json_body @validate( body_schema({ 'type': 'object', 'properties': { 'type': {'type': 'string'}, 'label': {'type': 'string'}, 'config': {'type': 'object'}, 'metadata': {'type': 'object'}, 'status_url': {'type': ['string', 'null']}, 'mo_url': {'type': ['string', 'null']}, 'rate_limit_count': { 'type': 'integer', 'minimum': 0, }, 'rate_limit_window': { 'type': 'integer', 'minimum': 0, }, 'character_limit': { 'type': 'integer', 'minimum': 0, }, }, })) @inlineCallbacks def modify_channel(self, request, body, channel_id): '''Mondify the channel configuration''' channel = yield Channel.from_id( self.redis, self.config, channel_id, self.service, self.plugins) resp = yield channel.update(body) returnValue(response( request, 'channel updated', resp)) @app.route('/channels/<string:channel_id>', methods=['DELETE']) @inlineCallbacks def delete_channel(self, request, channel_id): '''Delete the channel''' channel = yield Channel.from_id( self.redis, self.config, channel_id, self.service, self.plugins) yield channel.stop() yield channel.delete() returnValue(response( request, 'channel deleted', {})) @app.route('/channels/<string:channel_id>/restart', methods=['POST']) @inlineCallbacks def restart_channel(self, request, channel_id): '''Restart a channel.''' channel = yield Channel.from_id( self.redis, self.config, channel_id, self.service, self.plugins) yield channel.stop() yield channel.start(self.service) returnValue(response(request, 'channel restarted', {})) @app.route('/channels/<string:channel_id>/logs', methods=['GET']) @inlineCallbacks def get_logs(self, request, channel_id): '''Get the last N logs for a channel, sorted reverse chronologically.''' n = request.args.get('n', None) if n is not None: n = int(n[0]) channel = yield Channel.from_id( self.redis, self.config, channel_id, self.service, self.plugins) logs = yield channel.get_logs(n) returnValue(response(request, 'logs retrieved', logs)) @app.route('/channels/<string:channel_id>/messages/', methods=['POST']) @json_body @validate( body_schema({ 'type': 'object', 'properties': { 'to': {'type': 'string'}, 'from': {'type': ['string', 'null']}, 'group': {'type': ['string', 'null']}, 'reply_to': {'type': 'string'}, 'content': {'type': ['string', 'null']}, 'event_url': {'type': 'string'}, 'event_auth_token': {'type': 'string'}, 'priority': {'type': 'string'}, 'channel_data': {'type': 'object'}, }, 'required': ['content'], 'additionalProperties': False, })) @inlineCallbacks def send_message(self, request, body, channel_id): '''Send an outbound (mobile terminated) message''' if 'to' not in body and 'reply_to' not in body: raise ApiUsageError( 'Either "to" or "reply_to" must be specified') channel = yield Channel.from_id( self.redis, self.config, channel_id, self.service, self.plugins) if 'reply_to' in body: msg = yield channel.send_reply_message( self.message_sender, self.outbounds, self.inbounds, body, allow_expired_replies=self.config.allow_expired_replies) else: msg = yield channel.send_message( self.message_sender, self.outbounds, body) yield self.message_rate.increment( channel_id, 'outbound', self.config.metric_window) returnValue(response( request, 'message submitted', msg, code=http.CREATED)) @app.route( '/channels/<string:channel_id>/messages/<string:message_id>', methods=['GET']) @inlineCallbacks def get_message_status(self, request, channel_id, message_id): '''Retrieve the status of a message''' events = yield self.outbounds.load_all_events(channel_id, message_id) events = sorted( (api_from_event(channel_id, e) for e in events), key=lambda e: e['timestamp']) last_event = events[-1] if events else None last_event_type = last_event['event_type'] if last_event else None last_event_timestamp = last_event['timestamp'] if last_event else None returnValue(response(request, 'message status', { 'id': message_id, 'last_event_type': last_event_type, 'last_event_timestamp': last_event_timestamp, 'events': events, })) @app.route('/routers/', methods=['GET']) def get_router_list(self, request): """List all routers""" d = Router.get_all(self.router_store) d.addCallback(partial(response, request, 'routers retrieved')) return d @app.route('/routers/', methods=['POST']) @json_body @validate( body_schema({ 'type': 'object', 'properties': { 'type': {'type': 'string'}, 'label': {'type': 'string'}, 'config': {'type': 'object'}, 'metadata': {'type': 'object'}, }, 'additionalProperties': False, 'required': ['type', 'config'], })) @inlineCallbacks def create_router(self, request, body): """Create a new router""" router = Router(self, body) yield router.validate_config() router.start(self.service) yield router.save() returnValue(response( request, 'router created', (yield router.status()), code=http.CREATED )) @app.route('/routers/<string:router_id>', methods=['GET']) def get_router(self, request, router_id): """Get the configuration details and status of a specific router""" d = Router.from_id(self, router_id) d.addCallback(lambda router: router.status()) d.addCallback(partial(response, request, 'router found')) return d @app.route('/routers/<string:router_id>', methods=['PUT']) @json_body @validate( body_schema({ 'type': 'object', 'properties': { 'type': {'type': 'string'}, 'label': {'type': 'string'}, 'config': {'type': 'object'}, 'metadata': {'type': 'object'}, }, 'additionalProperties': False, 'required': ['type', 'config'], })) @inlineCallbacks def replace_router_config(self, request, body, router_id): """Replace the router config with the one specified""" router = yield Router.from_id(self, router_id) for field in ['type', 'label', 'config', 'metadata']: router.router_config.pop(field, None) router.router_config.update(body) yield router.validate_config() # Stop and start the router for the worker to get the new config yield router.stop() router.start(self.service) yield router.save() returnValue(response( request, 'router updated', (yield router.status()))) @app.route('/routers/<string:router_id>', methods=['PATCH']) @json_body @validate( body_schema({ 'type': 'object', 'properties': { 'type': {'type': 'string'}, 'label': {'type': 'string'}, 'config': {'type': 'object'}, 'metadata': {'type': 'object'}, }, 'additionalProperties': False, 'required': [], })) @inlineCallbacks def update_router_config(self, request, body, router_id): """Update the router config with the one specified""" router = yield Router.from_id(self, router_id) router.router_config.update(body) yield router.validate_config() # Stop and start the router for the worker to get the new config yield router.stop() router.start(self.service) yield router.save() returnValue(response( request, 'router updated', (yield router.status()))) @app.route('/routers/<string:router_id>', methods=['DELETE']) @inlineCallbacks def delete_router(self, request, router_id): router = yield Router.from_id(self, router_id) yield router.stop() yield router.delete() returnValue(response(request, 'router deleted', {})) @app.route('/routers/<string:router_id>/destinations/', methods=['POST']) @json_body @validate( body_schema({ 'type': 'object', 'properties': { 'label': {'type': 'string'}, 'config': {'type': 'object'}, 'metadata': {'type': 'object'}, 'mo_url': {'type': 'string'}, 'mo_url_token': {'type': 'string'}, 'amqp_queue': {'type': 'string'}, 'character_limit': { 'type': 'integer', 'minimum': 0, }, }, 'additionalProperties': False, 'required': ['config'], })) @inlineCallbacks def create_router_destination(self, request, body, router_id): """Create a new destination for the router""" router = yield Router.from_id(self, router_id) yield router.validate_destination_config(body['config']) destination = router.add_destination(body) yield router.stop() router.start(self.service) yield destination.save() returnValue(response( request, 'destination created', (yield destination.status()), code=http.CREATED )) @app.route('/routers/<string:router_id>/destinations/', methods=['GET']) def get_router_destination_list(self, request, router_id): """Get the list of destinations for a router""" d = Router.from_id(self, router_id) d.addCallback(lambda router: router.get_destination_list()) d.addCallback(partial(response, request, 'destinations retrieved')) return d @app.route( '/routers/<string:router_id>/destinations/<string:destination_id>', methods=['GET']) @inlineCallbacks def get_destination(self, request, router_id, destination_id): """Get the config and status of a destination""" router = yield Router.from_id(self, router_id) destination = router.get_destination(destination_id) returnValue(response( request, 'destination found', (yield destination.status()) )) @app.route( '/routers/<string:router_id>/destinations/<string:destination_id>', methods=['PUT']) @json_body @validate( body_schema({ 'type': 'object', 'properties': { 'label': {'type': 'string'}, 'config': {'type': 'object'}, 'metadata': {'type': 'object'}, 'mo_url': {'type': 'string'}, 'mo_url_token': {'type': 'string'}, 'amqp_queue': {'type': 'string'}, 'character_limit': { 'type': 'integer', 'minimum': 0, }, }, 'additionalProperties': False, 'required': ['config'], })) @inlineCallbacks def replace_router_destination( self, request, body, router_id, destination_id): """Replace the config of a router destination""" router = yield Router.from_id(self, router_id) yield router.validate_destination_config(body['config']) destination = router.get_destination(destination_id) destination.destination_config = body destination.destination_config['id'] = destination_id # Stop and start the router for the worker to get the new config yield router.stop() router.start(self.service) yield destination.save() returnValue(response( request, 'destination updated', (yield destination.status()))) @app.route( '/routers/<string:router_id>/destinations/<string:destination_id>', methods=['PATCH']) @json_body @validate( body_schema({ 'type': 'object', 'properties': { 'label': {'type': 'string'}, 'config': {'type': 'object'}, 'metadata': {'type': 'object'}, 'mo_url': {'type': 'string'}, 'mo_url_token': {'type': 'string'}, 'amqp_queue': {'type': 'string'}, 'character_limit': { 'type': 'integer', 'minimum': 0, }, }, 'additionalProperties': False, 'required': [], })) @inlineCallbacks def update_router_destination( self, request, body, router_id, destination_id): """Update the config of a router destination""" router = yield Router.from_id(self, router_id) if 'config' in body: yield router.validate_destination_config(body['config']) destination = router.get_destination(destination_id) destination.destination_config.update(body) # Stop and start the router for the worker to get the new config yield router.stop() router.start(self.service) yield destination.save() returnValue(response( request, 'destination updated', (yield destination.status()))) @app.route( '/routers/<string:router_id>/destinations/<string:destination_id>', methods=['DELETE']) @inlineCallbacks def delete_router_destination(self, request, router_id, destination_id): """Delete and stop the router destination""" router = yield Router.from_id(self, router_id) destination = router.get_destination(destination_id) yield router.stop() yield destination.delete() router.start(self.service) returnValue(response(request, 'destination deleted', {})) @app.route('/health', methods=['GET']) def health_status(self, request): if self.config.rabbitmq_management_interface: def get_queues(channel_ids): gets = [] for channel_id in channel_ids: for sub in ['inbound', 'outbound', 'event']: queue_name = "%s.%s" % (channel_id, sub) get = self.rabbitmq_management_client.get_queue( self.amqp_config['vhost'], queue_name) gets.append(get) return gets def return_queue_results(results): queues = [] stuck = False for result in results: queue = result[1] if ('messages' in queue): details = { 'name': queue['name'], 'stuck': False, 'messages': queue.get('messages'), 'rate': queue['messages_details']['rate'] } if (details['messages'] > 0 and details['rate'] == 0): stuck = True details['stuck'] = True queues.append(details) status = 'queues ok' code = http.OK if stuck: status = "queues stuck" code = http.INTERNAL_SERVER_ERROR return response(request, status, queues, code=code) d = Channel.get_all(self.redis) d.addCallback(get_queues) d.addCallback(defer.DeferredList) d.addCallback(return_queue_results) return d else: return response(request, 'health ok', {})
class MessageForwardingWorker(ApplicationWorker): '''This application worker consumes vumi messages placed on a configured amqp queue, and sends them as HTTP requests with a JSON body to a configured URL''' CONFIG_CLASS = MessageForwardingConfig @inlineCallbacks def setup_application(self): self.redis = yield TxRedisManager.from_config( self.config['redis_manager']) self.inbounds = InboundMessageStore(self.redis, self.config['inbound_ttl']) self.outbounds = OutboundMessageStore(self.redis, self.config['outbound_ttl']) self.message_rate = MessageRateStore(self.redis) if self.config.get('message_queue') is not None: self.ro_connector = yield self.setup_ro_connector( self.config['message_queue']) self.ro_connector.set_outbound_handler(self._publish_message) @inlineCallbacks def teardown_application(self): if getattr(self, 'redis', None) is not None: yield self.redis.close_manager() @property def channel_id(self): return self.config['transport_name'] @inlineCallbacks def consume_user_message(self, message): '''Sends the vumi message as an HTTP request to the configured URL''' yield self.inbounds.store_vumi_message(self.channel_id, message) msg = api_from_message(message) if self.config.get('mo_message_url') is not None: config = self.get_static_config() # Parse the basic auth username & password if available username = config.mo_message_url.username password = config.mo_message_url.password if any([username, password]): url = urlunparse(( config.mo_message_url.scheme, '%s%s' % (config.mo_message_url.hostname, (':%s' % (config.mo_message_url.port, ) if config.mo_message_url.port is not None else '')), config.mo_message_url.path, config.mo_message_url.params, config.mo_message_url.query, config.mo_message_url.fragment, )) auth = (username, password) else: url = config.mo_message_url.geturl() auth = None # Construct token auth headers if configured auth_token = config.mo_message_url_auth_token if auth_token: headers = {'Authorization': ['Token %s' % (auth_token, )]} else: headers = {} resp = yield post(url, msg, timeout=config.mo_message_url_timeout, auth=auth, headers=headers) if resp and request_failed(resp): logging.exception( 'Error sending message, received HTTP code %r with body %r' '. Message: %r' % (resp.code, (yield resp.content()), msg)) if self.config.get('message_queue') is not None: yield self.ro_connector.publish_inbound(message) yield self._increment_metric('inbound') @inlineCallbacks def store_and_forward_event(self, event): '''Store the event in the message store, POST it to the correct URL.''' yield self._store_event(event) yield self._forward_event(event) yield self._count_event(event) def _increment_metric(self, label): return self.message_rate.increment(self.channel_id, label, self.config['metric_window']) def _count_event(self, event): if event['event_type'] == 'ack': return self._increment_metric('submitted') if event['event_type'] == 'nack': return self._increment_metric('rejected') if event['event_type'] == 'delivery_report': if event['delivery_status'] == 'pending': return self._increment_metric('delivery_pending') if event['delivery_status'] == 'failed': return self._increment_metric('delivery_failed') if event['delivery_status'] == 'delivered': return self._increment_metric('delivery_succeeded') def _store_event(self, event): '''Stores the event in the message store''' message_id = event['user_message_id'] if message_id is None: logging.warning("Cannot store event, missing user_message_id: %r" % event) else: return self.outbounds.store_event(self.channel_id, message_id, event) @inlineCallbacks def _forward_event(self, event): '''Forward the event to the correct places.''' yield self._forward_event_http(event) yield self._forward_event_amqp(event) @inlineCallbacks def _forward_event_http(self, event): '''POST the event to the correct URL''' url = yield self._get_event_url(event) if url is None: return msg = api_from_event(self.channel_id, event) if msg['event_type'] is None: logging.exception("Discarding unrecognised event %r" % (event, )) return config = self.get_static_config() resp = yield post(url, msg, timeout=config.event_url_timeout) if resp and request_failed(resp): logging.exception( 'Error sending event, received HTTP code %r with body %r. ' 'Event: %r' % (resp.code, (yield resp.content()), event)) def _forward_event_amqp(self, event): '''Put the event on the correct queue.''' if self.config.get('message_queue') is not None: return self.ro_connector.publish_event(event) def consume_ack(self, event): return self.store_and_forward_event(event) def consume_nack(self, event): return self.store_and_forward_event(event) def consume_delivery_report(self, event): return self.store_and_forward_event(event) def _get_event_url(self, event): msg_id = event['user_message_id'] if msg_id is not None: return self.outbounds.load_event_url(self.channel_id, msg_id) else: logging.warning( "Cannot find event URL, missing user_message_id: %r" % event)
def create_store(self, **kw): ''' Creates and returns a new message rate store. ''' redis = yield self.get_redis() returnValue(MessageRateStore(redis, **kw))
class JunebugApi(object): app = Klein() def __init__(self, service, config): self.service = service self.redis_config = config.redis self.amqp_config = config.amqp self.config = config @inlineCallbacks def setup(self, redis=None, message_sender=None): if redis is None: redis = yield TxRedisManager.from_config(self.redis_config) if message_sender is None: message_sender = MessageSender('amqp-spec-0-8.xml', self.amqp_config) self.redis = redis self.message_sender = message_sender self.message_sender.setServiceParent(self.service) self.inbounds = InboundMessageStore(self.redis, self.config.inbound_message_ttl) self.outbounds = OutboundMessageStore(self.redis, self.config.outbound_message_ttl) self.message_rate = MessageRateStore(self.redis) self.plugins = [] for plugin_config in self.config.plugins: cls = load_class_by_string(plugin_config['type']) plugin = cls() yield plugin.start_plugin(plugin_config, self.config) self.plugins.append(plugin) yield Channel.start_all_channels(self.redis, self.config, self.service, self.plugins) @inlineCallbacks def teardown(self): yield self.redis.close_manager() for plugin in self.plugins: yield plugin.stop_plugin() @app.handle_errors(JunebugError) def generic_junebug_error(self, request, failure): return response(request, failure.value.description, { 'errors': [{ 'type': failure.value.name, 'message': failure.getErrorMessage(), }] }, code=failure.value.code) @app.handle_errors(HTTPException) def http_error(self, request, failure): error = { 'code': failure.value.code, 'type': failure.value.name, 'message': failure.getErrorMessage(), } if getattr(failure.value, 'new_url', None) is not None: request.setHeader('Location', failure.value.new_url) error['new_url'] = failure.value.new_url return response(request, failure.value.description, { 'errors': [error], }, code=failure.value.code) @app.handle_errors def generic_error(self, request, failure): log.err(failure) return response(request, 'generic error', { 'errors': [{ 'type': failure.type.__name__, 'message': failure.getErrorMessage(), }] }, code=http.INTERNAL_SERVER_ERROR) @app.route('/channels/', methods=['GET']) @inlineCallbacks def get_channel_list(self, request): '''List all channels''' ids = yield Channel.get_all(self.redis) returnValue(response(request, 'channels listed', sorted(ids))) @app.route('/channels/', methods=['POST']) @json_body @validate( body_schema({ 'type': 'object', 'properties': { 'type': { 'type': 'string' }, 'label': { 'type': 'string' }, 'config': { 'type': 'object' }, 'metadata': { 'type': 'object' }, 'status_url': { 'type': 'string' }, 'mo_url': { 'type': 'string' }, 'mo_url_auth_token': { 'type': 'string' }, 'amqp_queue': { 'type': 'string' }, 'rate_limit_count': { 'type': 'integer', 'minimum': 0, }, 'rate_limit_window': { 'type': 'integer', 'minimum': 0, }, 'character_limit': { 'type': 'integer', 'minimum': 0, }, }, 'required': ['type', 'config'], })) @inlineCallbacks def create_channel(self, request, body): '''Create a channel''' if not (body.get('mo_url') or body.get('amqp_queue')): raise ApiUsageError( 'One or both of "mo_url" and "amqp_queue" must be specified') channel = Channel(self.redis, self.config, body, self.plugins) yield channel.start(self.service) yield channel.save() returnValue( response(request, 'channel created', (yield channel.status()))) @app.route('/channels/<string:channel_id>', methods=['GET']) @inlineCallbacks def get_channel(self, request, channel_id): '''Return the channel configuration and a nested status object''' channel = yield Channel.from_id(self.redis, self.config, channel_id, self.service, self.plugins) resp = yield channel.status() returnValue(response(request, 'channel found', resp)) @app.route('/channels/<string:channel_id>', methods=['POST']) @json_body @validate( body_schema({ 'type': 'object', 'properties': { 'type': { 'type': 'string' }, 'label': { 'type': 'string' }, 'config': { 'type': 'object' }, 'metadata': { 'type': 'object' }, 'status_url': { 'type': ['string', 'null'] }, 'mo_url': { 'type': ['string', 'null'] }, 'rate_limit_count': { 'type': 'integer', 'minimum': 0, }, 'rate_limit_window': { 'type': 'integer', 'minimum': 0, }, 'character_limit': { 'type': 'integer', 'minimum': 0, }, }, })) @inlineCallbacks def modify_channel(self, request, body, channel_id): '''Mondify the channel configuration''' channel = yield Channel.from_id(self.redis, self.config, channel_id, self.service, self.plugins) resp = yield channel.update(body) returnValue(response(request, 'channel updated', resp)) @app.route('/channels/<string:channel_id>', methods=['DELETE']) @inlineCallbacks def delete_channel(self, request, channel_id): '''Delete the channel''' channel = yield Channel.from_id(self.redis, self.config, channel_id, self.service, self.plugins) yield channel.stop() yield channel.delete() returnValue(response(request, 'channel deleted', {})) @app.route('/channels/<string:channel_id>/restart', methods=['POST']) @inlineCallbacks def restart_channel(self, request, channel_id): '''Restart a channel.''' channel = yield Channel.from_id(self.redis, self.config, channel_id, self.service, self.plugins) yield channel.stop() yield channel.start(self.service) returnValue(response(request, 'channel restarted', {})) @app.route('/channels/<string:channel_id>/logs', methods=['GET']) @inlineCallbacks def get_logs(self, request, channel_id): '''Get the last N logs for a channel, sorted reverse chronologically.''' n = request.args.get('n', None) if n is not None: n = int(n[0]) channel = yield Channel.from_id(self.redis, self.config, channel_id, self.service, self.plugins) logs = yield channel.get_logs(n) returnValue(response(request, 'logs retrieved', logs)) @app.route('/channels/<string:channel_id>/messages/', methods=['POST']) @json_body @validate( body_schema({ 'type': 'object', 'properties': { 'to': { 'type': 'string' }, 'from': { 'type': ['string', 'null'] }, 'reply_to': { 'type': 'string' }, 'content': { 'type': ['string', 'null'] }, 'event_url': { 'type': 'string' }, 'priority': { 'type': 'string' }, 'channel_data': { 'type': 'object' }, }, 'required': ['content'], 'additionalProperties': False, })) @inlineCallbacks def send_message(self, request, body, channel_id): '''Send an outbound (mobile terminated) message''' if 'to' not in body and 'reply_to' not in body: raise ApiUsageError('Either "to" or "reply_to" must be specified') if 'to' in body and 'reply_to' in body: raise ApiUsageError( 'Only one of "to" and "reply_to" may be specified') if 'from' in body and 'reply_to' in body: raise ApiUsageError( 'Only one of "from" and "reply_to" may be specified') channel = yield Channel.from_id(self.redis, self.config, channel_id, self.service, self.plugins) if 'to' in body: msg = yield channel.send_message(self.message_sender, self.outbounds, body) else: msg = yield channel.send_reply_message(self.message_sender, self.outbounds, self.inbounds, body) yield self.message_rate.increment(channel_id, 'outbound', self.config.metric_window) returnValue(response(request, 'message sent', msg)) @app.route('/channels/<string:channel_id>/messages/<string:message_id>', methods=['GET']) @inlineCallbacks def get_message_status(self, request, channel_id, message_id): '''Retrieve the status of a message''' events = yield self.outbounds.load_all_events(channel_id, message_id) events = sorted((api_from_event(channel_id, e) for e in events), key=lambda e: e['timestamp']) last_event = events[-1] if events else None last_event_type = last_event['event_type'] if last_event else None last_event_timestamp = last_event['timestamp'] if last_event else None returnValue( response( request, 'message status', { 'id': message_id, 'last_event_type': last_event_type, 'last_event_timestamp': last_event_timestamp, 'events': events, })) @app.route('/health', methods=['GET']) def health_status(self, request): return response(request, 'health ok', {})
class JunebugApi(object): app = Klein() def __init__(self, service, config): self.service = service self.redis_config = config.redis self.amqp_config = config.amqp self.config = config @inlineCallbacks def setup(self, redis=None, message_sender=None): if redis is None: redis = yield TxRedisManager.from_config(self.redis_config) if message_sender is None: message_sender = MessageSender( 'amqp-spec-0-8.xml', self.amqp_config) self.redis = redis self.message_sender = message_sender self.message_sender.setServiceParent(self.service) self.inbounds = InboundMessageStore( self.redis, self.config.inbound_message_ttl) self.outbounds = OutboundMessageStore( self.redis, self.config.outbound_message_ttl) self.message_rate = MessageRateStore(self.redis) self.plugins = [] for plugin_config in self.config.plugins: cls = load_class_by_string(plugin_config['type']) plugin = cls() yield plugin.start_plugin(plugin_config, self.config) self.plugins.append(plugin) yield Channel.start_all_channels( self.redis, self.config, self.service, self.plugins) @inlineCallbacks def teardown(self): yield self.redis.close_manager() for plugin in self.plugins: yield plugin.stop_plugin() @app.handle_errors(JunebugError) def generic_junebug_error(self, request, failure): return response(request, failure.value.description, { 'errors': [{ 'type': failure.value.name, 'message': failure.getErrorMessage(), }] }, code=failure.value.code) @app.handle_errors(HTTPException) def http_error(self, request, failure): return response(request, failure.value.description, { 'errors': [{ 'type': failure.value.name, 'message': failure.getErrorMessage(), }] }, code=failure.value.code) @app.handle_errors def generic_error(self, request, failure): logging.exception(failure) return response(request, 'generic error', { 'errors': [{ 'type': failure.type.__name__, 'message': failure.getErrorMessage(), }] }, code=http.INTERNAL_SERVER_ERROR) @app.route('/channels/', methods=['GET']) @inlineCallbacks def get_channel_list(self, request): '''List all channels''' ids = yield Channel.get_all(self.redis) returnValue(response(request, 'channels listed', sorted(ids))) @app.route('/channels/', methods=['POST']) @json_body @validate( body_schema({ 'type': 'object', 'properties': { 'type': {'type': 'string'}, 'label': {'type': 'string'}, 'config': {'type': 'object'}, 'metadata': {'type': 'object'}, 'status_url': {'type': 'string'}, 'mo_url': {'type': 'string'}, 'amqp_queue': {'type': 'string'}, 'rate_limit_count': { 'type': 'integer', 'minimum': 0, }, 'rate_limit_window': { 'type': 'integer', 'minimum': 0, }, 'character_limit': { 'type': 'integer', 'minimum': 0, }, }, 'required': ['type', 'config'], })) @inlineCallbacks def create_channel(self, request, body): '''Create a channel''' if not (body.get('mo_url') or body.get('amqp_queue')): raise ApiUsageError( 'One or both of "mo_url" and "amqp_queue" must be specified') channel = Channel( self.redis, self.config, body, self.plugins) yield channel.save() yield channel.start(self.service) returnValue(response( request, 'channel created', (yield channel.status()))) @app.route('/channels/<string:channel_id>', methods=['GET']) @inlineCallbacks def get_channel(self, request, channel_id): '''Return the channel configuration and a nested status object''' channel = yield Channel.from_id( self.redis, self.config, channel_id, self.service, self.plugins) resp = yield channel.status() returnValue(response( request, 'channel found', resp)) @app.route('/channels/<string:channel_id>', methods=['POST']) @json_body @validate( body_schema({ 'type': 'object', 'properties': { 'type': {'type': 'string'}, 'label': {'type': 'string'}, 'config': {'type': 'object'}, 'metadata': {'type': 'object'}, 'status_url': {'type': 'string'}, 'mo_url': {'type': 'string'}, 'rate_limit_count': { 'type': 'integer', 'minimum': 0, }, 'rate_limit_window': { 'type': 'integer', 'minimum': 0, }, 'character_limit': { 'type': 'integer', 'minimum': 0, }, }, })) @inlineCallbacks def modify_channel(self, request, body, channel_id): '''Mondify the channel configuration''' channel = yield Channel.from_id( self.redis, self.config, channel_id, self.service, self.plugins) resp = yield channel.update(body) returnValue(response( request, 'channel updated', resp)) @app.route('/channels/<string:channel_id>', methods=['DELETE']) @inlineCallbacks def delete_channel(self, request, channel_id): '''Delete the channel''' channel = yield Channel.from_id( self.redis, self.config, channel_id, self.service, self.plugins) yield channel.stop() yield channel.delete() returnValue(response( request, 'channel deleted', {})) @app.route('/channels/<string:channel_id>/logs', methods=['GET']) @inlineCallbacks def get_logs(self, request, channel_id): '''Get the last N logs for a channel, sorted reverse chronologically.''' n = request.args.get('n', None) if n is not None: n = int(n[0]) channel = yield Channel.from_id( self.redis, self.config, channel_id, self.service, self.plugins) logs = yield channel.get_logs(n) returnValue(response(request, 'logs retrieved', logs)) @app.route('/channels/<string:channel_id>/messages/', methods=['POST']) @json_body @validate( body_schema({ 'type': 'object', 'properties': { 'to': {'type': 'string'}, 'from': {'type': ['string', 'null']}, 'reply_to': {'type': 'string'}, 'content': {'type': ['string', 'null']}, 'event_url': {'type': 'string'}, 'priority': {'type': 'string'}, 'channel_data': {'type': 'object'}, }, 'required': ['content'], 'additionalProperties': False, })) @inlineCallbacks def send_message(self, request, body, channel_id): '''Send an outbound (mobile terminated) message''' if 'to' not in body and 'reply_to' not in body: raise ApiUsageError( 'Either "to" or "reply_to" must be specified') if 'to' in body and 'reply_to' in body: raise ApiUsageError( 'Only one of "to" and "reply_to" may be specified') if 'from' in body and 'reply_to' in body: raise ApiUsageError( 'Only one of "from" and "reply_to" may be specified') channel = yield Channel.from_id( self.redis, self.config, channel_id, self.service, self.plugins) if 'to' in body: msg = yield channel.send_message( self.message_sender, self.outbounds, body) else: msg = yield channel.send_reply_message( self.message_sender, self.outbounds, self.inbounds, body) yield self.message_rate.increment( channel_id, 'outbound', self.config.metric_window) returnValue(response(request, 'message sent', msg)) @app.route( '/channels/<string:channel_id>/messages/<string:message_id>', methods=['GET']) @inlineCallbacks def get_message_status(self, request, channel_id, message_id): '''Retrieve the status of a message''' events = yield self.outbounds.load_all_events(channel_id, message_id) events = sorted( (api_from_event(channel_id, e) for e in events), key=lambda e: e['timestamp']) last_event = events[-1] if events else None last_event_type = last_event['event_type'] if last_event else None last_event_timestamp = last_event['timestamp'] if last_event else None returnValue(response(request, 'message status', { 'id': message_id, 'last_event_type': last_event_type, 'last_event_timestamp': last_event_timestamp, 'events': events, })) @app.route('/health', methods=['GET']) def health_status(self, request): return response(request, 'health ok', {})
class MessageForwardingWorker(ApplicationWorker): '''This application worker consumes vumi messages placed on a configured amqp queue, and sends them as HTTP requests with a JSON body to a configured URL''' CONFIG_CLASS = MessageForwardingConfig @inlineCallbacks def setup_application(self): self.redis = yield TxRedisManager.from_config( self.config['redis_manager']) self.inbounds = InboundMessageStore( self.redis, self.config['inbound_ttl']) self.outbounds = OutboundMessageStore( self.redis, self.config['outbound_ttl']) self.message_rate = MessageRateStore(self.redis) if self.config.get('message_queue') is not None: self.ro_connector = yield self.setup_ro_connector( self.config['message_queue']) self.ro_connector.set_outbound_handler( self._publish_message) @inlineCallbacks def teardown_application(self): yield self.redis.close_manager() @property def channel_id(self): return self.config['transport_name'] @inlineCallbacks def consume_user_message(self, message): '''Sends the vumi message as an HTTP request to the configured URL''' yield self.inbounds.store_vumi_message(self.channel_id, message) msg = api_from_message(message) if self.config.get('mo_message_url') is not None: resp = yield post(self.config['mo_message_url'], msg) if request_failed(resp): logging.exception( 'Error sending message, received HTTP code %r with body %r' '. Message: %r' % (resp.code, (yield resp.content()), msg)) if self.config.get('message_queue') is not None: yield self.ro_connector.publish_inbound(message) yield self._increment_metric('inbound') @inlineCallbacks def store_and_forward_event(self, event): '''Store the event in the message store, POST it to the correct URL.''' yield self._store_event(event) yield self._forward_event(event) yield self._count_event(event) def _increment_metric(self, label): return self.message_rate.increment( self.channel_id, label, self.config['metric_window']) def _count_event(self, event): if event['event_type'] == 'ack': return self._increment_metric('submitted') if event['event_type'] == 'nack': return self._increment_metric('rejected') if event['event_type'] == 'delivery_report': if event['delivery_status'] == 'pending': return self._increment_metric('delivery_pending') if event['delivery_status'] == 'failed': return self._increment_metric('delivery_failed') if event['delivery_status'] == 'delivered': return self._increment_metric('delivery_succeeded') def _store_event(self, event): '''Stores the event in the message store''' message_id = event['user_message_id'] return self.outbounds.store_event(self.channel_id, message_id, event) @inlineCallbacks def _forward_event(self, event): '''Forward the event to the correct places.''' yield self._forward_event_http(event) yield self._forward_event_amqp(event) @inlineCallbacks def _forward_event_http(self, event): '''POST the event to the correct URL''' url = yield self._get_event_url(event) if url is None: return msg = api_from_event(self.channel_id, event) if msg['event_type'] is None: logging.exception("Discarding unrecognised event %r" % (event,)) return resp = yield post(url, msg) if request_failed(resp): logging.exception( 'Error sending event, received HTTP code %r with body %r. ' 'Event: %r' % (resp.code, (yield resp.content()), event)) def _forward_event_amqp(self, event): '''Put the event on the correct queue.''' if self.config.get('message_queue') is not None: return self.ro_connector.publish_event(event) def consume_ack(self, event): return self.store_and_forward_event(event) def consume_nack(self, event): return self.store_and_forward_event(event) def consume_delivery_report(self, event): return self.store_and_forward_event(event) def _get_event_url(self, event): msg_id = event['user_message_id'] return self.outbounds.load_event_url(self.channel_id, msg_id)