def __init__(self, redis_manager, config, properties, plugins=[], id=None): '''Creates a new channel. ``redis_manager`` is the redis manager, from which a sub manager is created using the channel id. If the channel id is not supplied, a UUID one is generated. Call ``save`` to save the channel data. It can be started using the ``start`` function.''' self._properties = properties self.redis = redis_manager self.id = id self.config = config if self.id is None: self.id = str(uuid.uuid4()) self.options = deepcopy(VumiOptions.default_vumi_options) self.options.update(self.config.amqp) self.transport_worker = None self.application_worker = None self.status_application_worker = None self.sstore = StatusStore(self.redis) self.plugins = plugins self.message_rates = MessageRateStore(self.redis)
class ChannelStatusWorker(BaseWorker): '''This worker consumes status messages for the transport, and stores them in redis. Statuses with the same component are overwritten. It can also optionally forward the statuses to a URL''' CONFIG_CLASS = ChannelStatusConfig @inlineCallbacks def setup_connectors(self): connector = yield self.setup_receive_status_connector( "%s.status" % (self.config['channel_id'], )) connector.set_status_handler(self.consume_status) @inlineCallbacks def setup_worker(self): redis = yield TxRedisManager.from_config(self.config['redis_manager']) self.store = StatusStore(redis, ttl=None) yield self.unpause_connectors() def teardown_worker(self): pass @inlineCallbacks def consume_status(self, status): '''Store the status in redis under the correct component''' yield self.store.store_status(self.config['channel_id'], status) if self.config.get('status_url') is not None: yield self.send_status(status) @inlineCallbacks def send_status(self, status): data = api_from_status(self.config['channel_id'], status) config = self.get_static_config() resp = yield post(config.status_url, data, timeout=config.status_url_timeout) if resp and request_failed(resp): logging.exception( 'Error sending status event, received HTTP code %r with ' 'body %r. Status event: %r' % (resp.code, (yield resp.content()), status))
class ChannelStatusWorker(BaseWorker): '''This worker consumes status messages for the transport, and stores them in redis. Statuses with the same component are overwritten. It can also optionally forward the statuses to a URL''' CONFIG_CLASS = ChannelStatusConfig @inlineCallbacks def setup_connectors(self): connector = yield self.setup_receive_status_connector( "%s.status" % (self.config['channel_id'],)) connector.set_status_handler(self.consume_status) @inlineCallbacks def setup_worker(self): redis = yield TxRedisManager.from_config(self.config['redis_manager']) self.store = StatusStore(redis, ttl=None) yield self.unpause_connectors() def teardown_worker(self): pass @inlineCallbacks def consume_status(self, status): '''Store the status in redis under the correct component''' yield self.store.store_status(self.config['channel_id'], status) if self.config.get('status_url') is not None: yield self.send_status(status) @inlineCallbacks def send_status(self, status): data = api_from_status(self.config['channel_id'], status) config = self.get_static_config() resp = yield post(config.status_url, data, timeout=config.status_url_timeout) if resp and request_failed(resp): logging.exception( 'Error sending status event, received HTTP code %r with ' 'body %r. Status event: %r' % (resp.code, (yield resp.content()), status))
def setup_worker(self): redis = yield TxRedisManager.from_config(self.config['redis_manager']) self.store = StatusStore(redis, ttl=None) yield self.unpause_connectors()
class Channel(object): OUTBOUND_QUEUE = '%s.outbound' APPLICATION_ID = 'application:%s' STATUS_APPLICATION_ID = 'status:%s' APPLICATION_CLS_NAME = 'junebug.workers.MessageForwardingWorker' STATUS_APPLICATION_CLS_NAME = 'junebug.workers.ChannelStatusWorker' JUNEBUG_LOGGING_SERVICE_CLS = JunebugLoggerService def __init__(self, redis_manager, config, properties, plugins=[], id=None): '''Creates a new channel. ``redis_manager`` is the redis manager, from which a sub manager is created using the channel id. If the channel id is not supplied, a UUID one is generated. Call ``save`` to save the channel data. It can be started using the ``start`` function.''' self._properties = properties self.redis = redis_manager self.id = id self.config = config if self.id is None: self.id = str(uuid.uuid4()) self.options = deepcopy(VumiOptions.default_vumi_options) self.options.update(self.config.amqp) self.transport_worker = None self.application_worker = None self.status_application_worker = None self.sstore = StatusStore(self.redis) self.plugins = plugins self.message_rates = MessageRateStore(self.redis) @property def application_id(self): return self.APPLICATION_ID % (self.id,) @property def status_application_id(self): return self.STATUS_APPLICATION_ID % (self.id,) @property def character_limit(self): return self._properties.get('character_limit') @inlineCallbacks def start(self, service, transport_worker=None): '''Starts the relevant workers for the channel. ``service`` is the parent of under which the workers should be started.''' self._start_transport(service, transport_worker) self._start_application(service) self._start_status_application(service) for plugin in self.plugins: yield plugin.channel_started(self) @inlineCallbacks def stop(self): '''Stops the relevant workers for the channel''' yield self._stop_application() yield self._stop_status_application() yield self._stop_transport() for plugin in self.plugins: yield plugin.channel_stopped(self) @inlineCallbacks def save(self): '''Saves the channel data into redis.''' properties = json.dumps(self._properties) channel_redis = yield self.redis.sub_manager(self.id) yield channel_redis.set('properties', properties) yield self.redis.sadd('channels', self.id) @inlineCallbacks def update(self, properties): '''Updates the channel configuration, saves the updated configuration, and (if needed) restarts the channel with the new configuration. Returns the updated configuration and status.''' self._properties.update(properties) yield self.save() service = self.transport_worker.parent # Only restart if the channel config has changed if 'config' in properties: yield self._stop_transport() yield self._start_transport(service) if 'mo_url' in properties: yield self._stop_application() yield self._start_application(service) returnValue((yield self.status())) @inlineCallbacks def delete(self): '''Removes the channel data from redis''' channel_redis = yield self.redis.sub_manager(self.id) yield channel_redis.delete('properties') yield self.redis.srem('channels', self.id) @inlineCallbacks def status(self): '''Returns a dict with the configuration and status of the channel''' status = deepcopy(self._properties) status['id'] = self.id status['status'] = yield self._get_status() returnValue(status) def _get_message_rate(self, label): return self.message_rates.get_messages_per_second( self.id, label, self.config.metric_window) @inlineCallbacks def _get_status(self): components = yield self.sstore.get_statuses(self.id) components = dict( (k, api_from_status(self.id, v)) for k, v in components.iteritems() ) status_values = { 'down': 0, 'degraded': 1, 'ok': 2, } try: status = min( (c['status'] for c in components.values()), key=status_values.get) except ValueError: # No statuses status = None returnValue({ 'components': components, 'status': status, 'inbound_message_rate': ( yield self._get_message_rate('inbound')), 'outbound_message_rate': ( yield self._get_message_rate('outbound')), 'submitted_event_rate': ( yield self._get_message_rate('submitted')), 'rejected_event_rate': ( yield self._get_message_rate('rejected')), 'delivery_succeeded_rate': ( yield self._get_message_rate('delivery_succeeded')), 'delivery_failed_rate': ( yield self._get_message_rate('delivery_failed')), 'delivery_pending_rate': ( yield self._get_message_rate('delivery_pending')), }) @classmethod @inlineCallbacks def from_id(cls, redis, config, id, parent, plugins=[]): '''Creates a channel by loading the data from redis, given the channel's id, and the parent service of the channel''' channel_redis = yield redis.sub_manager(id) properties = yield channel_redis.get('properties') if properties is None: raise ChannelNotFound() properties = json.loads(properties) obj = cls(redis, config, properties, plugins, id=id) obj._restore(parent) returnValue(obj) @classmethod @inlineCallbacks def get_all(cls, redis): '''Returns a set of keys of all of the channels''' channels = yield redis.smembers('channels') returnValue(channels) @classmethod @inlineCallbacks def start_all_channels(cls, redis, config, parent, plugins=[]): '''Ensures that all of the stored channels are running''' for id in (yield cls.get_all(redis)): if id not in parent.namedServices: properties = json.loads(( yield redis.get('%s:properties' % id))) channel = cls(redis, config, properties, plugins, id=id) yield channel.start(parent) @inlineCallbacks def send_message(self, sender, outbounds, msg): '''Sends a message.''' event_url = msg.get('event_url') msg = message_from_api(self.id, msg) msg = TransportUserMessage.send(**msg) msg = yield self._send_message(sender, outbounds, event_url, msg) returnValue(api_from_message(msg)) @inlineCallbacks def send_reply_message(self, sender, outbounds, inbounds, msg): '''Sends a reply message.''' in_msg = yield inbounds.load_vumi_message(self.id, msg['reply_to']) if in_msg is None: raise MessageNotFound( "Inbound message with id %s not found" % (msg['reply_to'],)) event_url = msg.get('event_url') msg = message_from_api(self.id, msg) msg = in_msg.reply(**msg) msg = yield self._send_message(sender, outbounds, event_url, msg) returnValue(api_from_message(msg)) def get_logs(self, n): '''Returns the last `n` logs. If `n` is greater than the configured limit, only returns the configured limit amount of logs. If `n` is None, returns the configured limit amount of logs.''' if n is None: n = self.config.max_logs n = min(n, self.config.max_logs) logfile = self.transport_worker.getServiceNamed( 'Junebug Worker Logger').logfile return read_logs(logfile, n) @property def _transport_config(self): config = self._properties['config'] config = self._convert_unicode(config) config['transport_name'] = self.id config['worker_name'] = self.id config['publish_status'] = True return config @property def _application_config(self): return { 'transport_name': self.id, 'mo_message_url': self._properties['mo_url'], 'redis_manager': self.config.redis, 'inbound_ttl': self.config.inbound_message_ttl, 'outbound_ttl': self.config.outbound_message_ttl, 'metric_window': self.config.metric_window, } @property def _status_application_config(self): return { 'redis_manager': self.config.redis, 'channel_id': self.id, 'status_url': self._properties.get('status_url'), } @property def _available_transports(self): if self.config.replace_channels: return self.config.channels else: channels = {} channels.update(transports) channels.update(self.config.channels) return channels @property def _transport_cls_name(self): cls_name = self._available_transports.get(self._properties.get('type')) if cls_name is None: raise InvalidChannelType( 'Invalid channel type %r, must be one of: %s' % ( self._properties.get('type'), ', '.join(self._available_transports.keys()))) return cls_name def _start_transport(self, service, transport_worker=None): # transport_worker parameter is for testing, if it is None, # create the transport worker if transport_worker is None: transport_worker = self._create_transport() transport_worker.setName(self.id) logging_service = self._create_junebug_logger_service() transport_worker.addService(logging_service) transport_worker.setServiceParent(service) self.transport_worker = transport_worker def _start_application(self, service): worker = self._create_application() worker.setName(self.application_id) worker.setServiceParent(service) self.application_worker = worker def _start_status_application(self, service): worker = self._create_status_application() worker.setName(self.status_application_id) worker.setServiceParent(service) self.status_application_worker = worker def _create_transport(self): return self._create_worker( self._transport_cls_name, self._transport_config) def _create_application(self): return self._create_worker( self.APPLICATION_CLS_NAME, self._application_config) def _create_status_application(self): return self._create_worker( self.STATUS_APPLICATION_CLS_NAME, self._status_application_config) def _create_junebug_logger_service(self): return self.JUNEBUG_LOGGING_SERVICE_CLS( self.id, self.config.logging_path, self.config.log_rotate_size, self.config.max_log_files) def _create_worker(self, cls_name, config): creator = WorkerCreator(self.options) worker = creator.create_worker(cls_name, config) return worker @inlineCallbacks def _stop_transport(self): if self.transport_worker is not None: yield self.transport_worker.disownServiceParent() self.transport_worker = None @inlineCallbacks def _stop_application(self): if self.application_worker is not None: yield self.application_worker.disownServiceParent() self.application_worker = None @inlineCallbacks def _stop_status_application(self): if self.status_application_worker is not None: yield self.status_application_worker.disownServiceParent() self.status_application_worker = None def _restore(self, service): self.transport_worker = service.getServiceNamed(self.id) self.application_worker = service.getServiceNamed(self.application_id) self.status_application_worker = service.getServiceNamed( self.status_application_id) def _convert_unicode(self, data): # Twisted doesn't like it when we give unicode in for config things if isinstance(data, basestring): return str(data) elif isinstance(data, collections.Mapping): return dict(map(self._convert_unicode, data.iteritems())) elif isinstance(data, collections.Iterable): return type(data)(map(self._convert_unicode, data)) else: return data def _check_character_limit(self, content): count = len(content) if (self.character_limit is not None and count > self.character_limit): raise MessageTooLong( 'Message content %r is of length %d, which is greater than the' ' character limit of %d' % ( content, count, self.character_limit) ) @inlineCallbacks def _send_message(self, sender, outbounds, event_url, msg): self._check_character_limit(msg['content']) if event_url is not None: yield outbounds.store_event_url( self.id, msg['message_id'], event_url) queue = self.OUTBOUND_QUEUE % (self.id,) msg = yield sender.send_message(msg, routing_key=queue) returnValue(msg)
def create_store(self): redis = yield self.get_redis() store = StatusStore(redis, ttl=None) returnValue(store)