def setup(self, redis=None, message_sender=None): if redis is None: redis = yield TxRedisManager.from_config(self.redis_config) if message_sender is None: message_sender = MessageSender('amqp-spec-0-8.xml', self.amqp_config) self.redis = redis self.message_sender = message_sender self.message_sender.setServiceParent(self.service) self.inbounds = InboundMessageStore(self.redis, self.config.inbound_message_ttl) self.outbounds = OutboundMessageStore(self.redis, self.config.outbound_message_ttl) self.message_rate = MessageRateStore(self.redis) self.plugins = [] for plugin_config in self.config.plugins: cls = load_class_by_string(plugin_config['type']) plugin = cls() yield plugin.start_plugin(plugin_config, self.config) self.plugins.append(plugin) yield Channel.start_all_channels(self.redis, self.config, self.service, self.plugins)
def setup(self, redis=None, message_sender=None): if redis is None: redis = yield TxRedisManager.from_config(self.redis_config) if message_sender is None: message_sender = MessageSender( 'amqp-spec-0-8.xml', self.amqp_config) self.redis = redis self.message_sender = message_sender self.message_sender.setServiceParent(self.service) self.inbounds = InboundMessageStore( self.redis, self.config.inbound_message_ttl) self.outbounds = OutboundMessageStore( self.redis, self.config.outbound_message_ttl) self.message_rate = MessageRateStore(self.redis) self.plugins = [] for plugin_config in self.config.plugins: cls = load_class_by_string(plugin_config['type']) plugin = cls() yield plugin.start_plugin(plugin_config, self.config) self.plugins.append(plugin) yield Channel.start_all_channels( self.redis, self.config, self.service, self.plugins)
def setup_routing(self): self.r_config = self.config.get('redis_manager', {}) self.r_prefix = self.config['dispatcher_name'] self.rules = [] for rule in self.config.get('rules', []): if 'keyword' not in rule or 'app' not in rule: raise ConfigError("Rule definition %r must contain values for" " both 'app' and 'keyword'" % rule) rule = rule.copy() rule['keyword'] = rule['keyword'].lower() self.rules.append(rule) keyword_mappings = self.config.get('keyword_mappings', {}) for transport_name, keyword in keyword_mappings.items(): self.rules.append({'app': transport_name, 'keyword': keyword.lower()}) self.fallback_application = self.config.get('fallback_application') self.transport_mappings = self.config['transport_mappings'] self.expire_routing_timeout = int(self.config.get( 'expire_routing_memory', self.DEFAULT_ROUTING_TIMEOUT)) # FIXME: The following is a hack to deal with sync-only setup. self._redis_d = TxRedisManager.from_config(self.r_config) self._redis_d.addCallback(lambda m: m.sub_manager(self.r_prefix)) self._redis_d.addCallback(self._setup_redis)
def setup(self): self.r_config = self.config.get('redis_manager', {}) self.keys_per_user_hard = self.config.get( 'keys_per_user_hard', self.config.get('keys_per_user', 100)) self.keys_per_user_soft = self.config.get( 'keys_per_user_soft', int(0.8 * self.keys_per_user_hard)) self.redis = yield TxRedisManager.from_config(self.r_config)
def setup_routing(self): self.r_config = self.config.get('redis_manager', {}) self.r_prefix = self.config['dispatcher_name'] self.rules = [] for rule in self.config.get('rules', []): if 'keyword' not in rule or 'app' not in rule: raise ConfigError("Rule definition %r must contain values for" " both 'app' and 'keyword'" % rule) rule = rule.copy() rule['keyword'] = rule['keyword'].lower() self.rules.append(rule) keyword_mappings = self.config.get('keyword_mappings', {}) for transport_name, keyword in keyword_mappings.items(): self.rules.append({ 'app': transport_name, 'keyword': keyword.lower() }) self.fallback_application = self.config.get('fallback_application') self.transport_mappings = self.config['transport_mappings'] self.expire_routing_timeout = int( self.config.get('expire_routing_memory', self.DEFAULT_ROUTING_TIMEOUT)) # FIXME: The following is a hack to deal with sync-only setup. self._redis_d = TxRedisManager.from_config(self.r_config) self._redis_d.addCallback(lambda m: m.sub_manager(self.r_prefix)) self._redis_d.addCallback(self._setup_redis)
def setup_transport(self): config = self.get_static_config() log.msg('Starting SMPP Transport for: %s' % (config.twisted_endpoint,)) default_prefix = '%s@%s' % (config.system_id, config.transport_name) redis_prefix = config.split_bind_prefix or default_prefix self.redis = (yield TxRedisManager.from_config( config.redis_manager)).sub_manager(redis_prefix) self.dr_processor = config.delivery_report_processor( self, config.delivery_report_processor_config) self.deliver_sm_processor = config.deliver_short_message_processor( self, config.deliver_short_message_processor_config) self.submit_sm_processor = config.submit_short_message_processor( self, config.submit_short_message_processor_config) self.sequence_generator = self.sequence_class(self.redis) self.message_stash = SmppMessageDataStash(self.redis, config) self.throttled = None self._throttled_message_ids = [] self._unthrottle_delayedCall = None self.factory = self.factory_class(self) self.service = self.start_service(self.factory) self.tps_counter = 0 self.tps_limit = config.mt_tps if config.mt_tps > 0: self.mt_tps_lc = LoopingCall(self.reset_mt_tps) self.mt_tps_lc.clock = self.clock self.mt_tps_lc.start(1, now=True) else: self.mt_tps_lc = None
def setup_transport(self): config = self.get_static_config() self.redis = yield TxRedisManager.from_config(config.redis_manager) self.retries = 0 self.delay = config.initial_delay self.reconnect_call = None self.client = StreamingClient(self.agent_factory) self.connect_api_clients()
def setup(self): super(GoLoggingResource, self).setup() redis_config = self.config.get('redis_manager', {}) max_logs_per_conversation = self.config.get( 'max_logs_per_conversation') self._redis = yield TxRedisManager.from_config(redis_config) self.log_manager = LogManager( self._redis, max_logs_per_conversation=max_logs_per_conversation)
def setup_transport(self): config = self.get_static_config() self.redis = yield TxRedisManager.from_config(config.redis_manager) self.retries = 0 self.delay = config.initial_delay self.reconnect_call = None self.client = StreamingClient() self.connect_api_clients()
def get_manager(self): manager = yield TxRedisManager.from_config({ 'FAKE_REDIS': 'yes', 'key_prefix': 'redistest', }) self.add_cleanup(self.cleanup_manager, manager) yield manager._purge_all() returnValue(manager)
def setup_middleware(self): store_prefix = self.config.store_prefix r_config = self.config.redis_manager self.redis = yield TxRedisManager.from_config(r_config) manager = TxRiakManager.from_config(self.config.riak_manager) self.store = MessageStore(manager, self.redis.sub_manager(store_prefix)) self.store_on_consume = self.config.store_on_consume
def setup_worker(self): config = self.get_static_config() redis_manager = yield TxRedisManager.from_config(config.redis_manager) lop_manager = LopManager(redis_manager) rpc = LopApiServer(lop_manager) addIntrospection(rpc) site = Site(rpc) self.addService(strports.service(config.twisted_endpoint, site))
def setup_middleware(self): self.validate_config() self.metric_publisher = yield self.worker.start_publisher(MetricPublisher) # We don't use a VumiApi here because we don't have a Riak config for # it. self.redis = yield TxRedisManager.from_config(self.config["redis_manager"]) self.metric_manager = MetricManager(self.manager_name + ".", publisher=self.metric_publisher) self.metric_manager.start_polling()
def setup_middleware(self): store_prefix = self.config.get('store_prefix', 'message_store') r_config = self.config.get('redis_manager', {}) self.redis = yield TxRedisManager.from_config(r_config) manager = TxRiakManager.from_config(self.config.get('riak_manager')) self.store = MessageStore(manager, self.redis.sub_manager(store_prefix)) self.store_on_consume = self.config.get('store_on_consume', True)
def setUp(self): config = { 'key_prefix': 'heartbeats', 'db': 5, 'FAKE_REDIS': True, } self.redis = yield TxRedisManager.from_config(config) self.add_cleanup(self.cleanup_redis) self.stg = storage.Storage(self.redis)
def from_redis_config(cls, config, key_prefix=None, max_session_length=None, gc_period=None): """Create a `SessionManager` instance using `TxRedisManager`. """ from vumi.persist.txredis_manager import TxRedisManager d = TxRedisManager.from_config(config) if key_prefix is not None: d.addCallback(lambda m: m.sub_manager(key_prefix)) return d.addCallback(lambda m: cls(m, max_session_length, gc_period))
def setUp(self): self.manager = yield TxRedisManager.from_config({ 'FAKE_REDIS': 'yes', 'key_prefix': 'redistest' }) self.add_cleanup(self.cleanup_manager) yield self.manager._purge_all()
def setup_application(self): config = self.get_static_config() self.redis = None if config.allow_replies: self.redis = yield TxRedisManager.from_config(config.redis_manager) send_resource = self.get_protected_resource(SendResource(self)) self.web_resource = yield self.start_web_resources( [(send_resource, config.web_path), (HealthResource(), "health")], config.web_port )
def from_config_async(cls, config, command_publisher=None, metric_publisher=None): # Note: This takes a publisher rather than a client to avoid leaking # AMQP channels by making our own transient publishers. riak_config, redis_config = cls._parse_config(config) manager = TxRiakManager.from_config(riak_config) redis = yield TxRedisManager.from_config(redis_config) sender = AsyncMessageSender(command_publisher) returnValue(cls(manager, redis, sender, metric_publisher))
def setup_application(self): self.redis = yield TxRedisManager.from_config( self.config['redis_manager']) self.inbounds = InboundMessageStore( self.redis, self.config['inbound_ttl']) self.outbounds = OutboundMessageStore( self.redis, self.config['outbound_ttl'])
def setup_middleware(self): self.redis = yield TxRedisManager.from_config( self.config.redis_manager) self.timeout = self.config.timeout self.field_name = self.config.field_name self.clock = reactor namespace_type = self.config.namespace_type self.namespace_handler = getattr( self, self.NAMESPACE_HANDLERS[namespace_type])
def setup_routing(self): r_config = self.config.get('redis_manager', {}) r_prefix = self.config['dispatcher_name'] # FIXME: The following is a hack to deal with sync-only setup. self._redis_d = TxRedisManager.from_config(r_config) self._redis_d.addCallback(lambda m: m.sub_manager(r_prefix)) self._redis_d.addCallback(self._setup_redis) self.groups = self.config['group_mappings'] self.nr_of_groups = len(self.groups)
def setup_application(self): config = self.get_static_config() self.redis = None if config.allow_replies: self.redis = yield TxRedisManager.from_config(config.redis_manager) send_resource = self.get_protected_resource(SendResource(self)) self.web_resource = yield self.start_web_resources([ (send_resource, config.web_path), (HealthResource(), 'health'), ], config.web_port)
def setup_middleware(self): self.validate_config() self.metric_publisher = yield self.worker.start_publisher( MetricPublisher) # We don't use a VumiApi here because we don't have a Riak config for # it. self.redis = yield TxRedisManager.from_config( self.config['redis_manager']) self.metric_manager = MetricManager(self.manager_name + '.', publisher=self.metric_publisher) self.metric_manager.start_polling()
def setup_transport(self): config = self.get_static_config() self.redis = yield TxRedisManager.from_config(config.redis_manager) self.web_resource = yield self.start_web_resources([ (GoConversationResource( self.handle_raw_inbound_message), config.message_path), (GoConversationResource( self.handle_raw_inbound_event), config.event_path), (GoConversationHealthResource(self), config.health_path), ], config.web_port)
def setup_worker(self): config = self.get_static_config() riak = yield TxRiakManager.from_config(config.riak_manager) redis = yield TxRedisManager.from_config(config.redis_manager) self.store = MessageStore(riak, redis) site = build_web_site({ config.web_path: MessageStoreResource(self.store), config.health_path: httprpc.HttpRpcHealthResource(self), }) self.addService( StreamServerEndpointService(config.twisted_endpoint, site))
def setup_worker(self): config = self.get_static_config() self._riak = yield TxRiakManager.from_config(config.riak_manager) redis = yield TxRedisManager.from_config(config.redis_manager) self.store = MessageStore(self._riak, redis) site = build_web_site({ config.web_path: MessageStoreResource(self.store), config.health_path: httprpc.HttpRpcHealthResource(self), }) self.addService( StreamServerEndpointService(config.twisted_endpoint, site))
def get_redis(self, **kwargs): from vumi.persist.txredis_manager import TxRedisManager # Fake redis fake_redis = yield TxRedisManager._fake_manager( FakeRedis(async=True, **kwargs), { "config": {}, "key_prefix": 'redistest', }) self.add_cleanup(fake_redis._close) # Real redis config = { 'FAKE_REDIS': 'yes', 'key_prefix': 'redistest', } config.update(kwargs) real_redis = yield TxRedisManager.from_config(config) self.add_cleanup(self.cleanup_manager, real_redis) # Both redises yield real_redis._purge_all() returnValue(RedisPairWrapper(self, fake_redis, real_redis))
def setup_transport(self): config = self.get_static_config() self.redis = yield TxRedisManager.from_config(config.redis_manager) self.web_resource = yield self.start_web_resources( [ (GoConversationResource(self.handle_raw_inbound_message), config.message_path), (GoConversationResource(self.handle_raw_inbound_event), config.event_path), (GoConversationHealthResource(self), config.health_path), ], config.web_port, )
def startWorker(self): web_path = self.config['web_path'] web_port = int(self.config['web_port']) health_path = self.config['health_path'] riak = yield TxRiakManager.from_config(self.config['riak_manager']) redis = yield TxRedisManager.from_config(self.config['redis_manager']) self.store = MessageStore(riak, redis) self.webserver = self.start_web_resources([ (MessageStoreAPI(self.store), web_path), (httprpc.HttpRpcHealthResource(self), health_path), ], web_port)
def setup_transport(self): self.setup_cacerts() config = self.get_static_config() self.redis = yield TxRedisManager.from_config(config.redis_manager) self.web_resource = yield self.start_web_resources([ (GoConversationResource(self.handle_raw_inbound_message), "%s/messages.json" % (config.web_path)), (GoConversationResource(self.handle_raw_inbound_event), "%s/events.json" % (config.web_path)), (GoConversationHealthResource(self), config.health_path), ], config.web_port) self.status_detect = StatusEdgeDetector()
def setup_worker(self): config = self.get_static_config() self.redis_manager = yield TxRedisManager.from_config( config.redis_manager) tagpool = TagpoolManager(self.redis_manager) rpc = TagpoolApiServer(tagpool) addIntrospection(rpc) site = build_web_site({ config.web_path: rpc, config.health_path: httprpc.HttpRpcHealthResource(self), }) self.addService( StreamServerEndpointService(config.twisted_endpoint, site))
def setup_router(self): config = self.get_static_config() self.redis = yield TxRedisManager.from_config( self.config['redis_manager']) self.outbounds = OutboundMessageStore( self.redis, self.config['outbound_ttl']) yield self.consume_channel( str(config.channel), self.handle_inbound_message, self.handle_inbound_event) for destination in config.destinations: self.consume_destination( destination['id'], self.handle_outbound_message)
def setup_transport(self): self.setup_cacerts() config = self.get_static_config() self.redis = yield TxRedisManager.from_config( config.redis_manager) self.web_resource = yield self.start_web_resources([ (GoConversationResource(self.handle_raw_inbound_message), "%s/messages.json" % (config.web_path)), (GoConversationResource(self.handle_raw_inbound_event), "%s/events.json" % (config.web_path)), (GoConversationHealthResource(self), config.health_path), ], config.web_port) self.status_detect = StatusEdgeDetector()
def setup_transport(self): config = self.get_static_config() self.request_dict = {} self.endpoint = config.twisted_endpoint self.resource = WeChatResource(self) self.factory = build_web_site({config.health_path: HttpRpcHealthResource(self), config.web_path: self.resource}) self.redis = yield TxRedisManager.from_config(config.redis_manager) self.server = yield self.endpoint.listen(self.factory) if config.wechat_menu: # not yielding because this shouldn't block startup d = self.get_access_token() d.addCallback(self.create_wechat_menu, config.wechat_menu)
def setup_application(self): """Application specific setup""" self.app_config = self.get_static_config() self.server = TwilioAPIServer(self, self.app_config.api_version) path = os.path.join( self.app_config.web_path, self.app_config.api_version) self.webserver = self.start_web_resources([ (self.server.app.resource(), path)], self.app_config.web_port) redis = yield TxRedisManager.from_config(self.app_config.redis_manager) self.session_manager = SessionManager( redis, self.app_config.redis_timeout) self.session_lookup = SessionIDLookup( redis, self.app_config.redis_timeout, self.app_config.session_lookup_namespace)
def setup_application(self): self.redis = yield TxRedisManager.from_config( self.config['redis_manager']) self.inbounds = InboundMessageStore(self.redis, self.config['inbound_ttl']) self.outbounds = OutboundMessageStore(self.redis, self.config['outbound_ttl']) self.message_rate = MessageRateStore(self.redis) if self.config.get('message_queue') is not None: self.ro_connector = yield self.setup_ro_connector( self.config['message_queue']) self.ro_connector.set_outbound_handler(self._publish_message)
def get_redis(self, **kwargs): from vumi.persist.txredis_manager import TxRedisManager # Fake redis fake_redis = FakeRedis(async=True, **kwargs) self.add_cleanup(fake_redis.teardown) # Real redis config = { 'FAKE_REDIS': 'yes', 'key_prefix': 'redistest', } config.update(kwargs) real_redis = yield TxRedisManager.from_config(config) self.add_cleanup(self.cleanup_manager, real_redis) # Both redises yield real_redis._purge_all() returnValue(RedisPairWrapper(self, fake_redis, real_redis))
def setup_application(self): config = self.get_static_config() yield self._setup_metrics(config.metrics_prefix) redis = yield TxRedisManager.from_config(config.redis_manager) self._redis = redis.sub_manager(self.config['worker_name']) if config.content_cache_time: self.extract_redis = redis.sub_manager('extracts') self.connectors[self.transport_name].set_inbound_handler( self.consume_content_sms_message, 'sms_content') self.connectors[self.transport_name].set_event_handler( self.consume_content_sms_event, 'sms_content') self.hash_algorithm = getattr(hashlib, config.hash_algorithm) self.secret_key = config.secret_key
def setup_application(self): self.redis = yield TxRedisManager.from_config( self.config['redis_manager']) self.inbounds = InboundMessageStore( self.redis, self.config['inbound_ttl']) self.outbounds = OutboundMessageStore( self.redis, self.config['outbound_ttl']) self.message_rate = MessageRateStore(self.redis) if self.config.get('message_queue') is not None: self.ro_connector = yield self.setup_ro_connector( self.config['message_queue']) self.ro_connector.set_outbound_handler( self._publish_message)
def setup(self, redis=None, message_sender=None): if redis is None: redis = yield TxRedisManager.from_config(self.redis_config) if message_sender is None: message_sender = MessageSender( 'amqp-spec-0-8.xml', self.amqp_config) self.redis = redis self.message_sender = message_sender self.message_sender.setServiceParent(self.service) self.inbounds = InboundMessageStore( self.redis, self.config.inbound_message_ttl) self.outbounds = OutboundMessageStore( self.redis, self.config.outbound_message_ttl)
def setup_transport(self): config = self.get_static_config() self.request_dict = {} self.endpoint = config.twisted_endpoint self.resource = WeChatResource(self) self.factory = build_web_site({ config.health_path: HttpRpcHealthResource(self), config.web_path: self.resource, }) self.redis = yield TxRedisManager.from_config(config.redis_manager) self.server = yield self.endpoint.listen(self.factory) if config.wechat_menu: # not yielding because this shouldn't block startup d = self.get_access_token() d.addCallback(self.create_wechat_menu, config.wechat_menu)
def startWorker(self): log.msg("Heartbeat monitor initializing") config = self.get_static_config() self.deadline = config.deadline redis_config = config.redis_manager self._redis = yield TxRedisManager.from_config(redis_config) self._storage = Storage(self._redis) self._systems, self._workers = self.parse_config( config.monitored_systems) # Start consuming heartbeats yield self.consume("heartbeat.inbound", self._consume_message, exchange_name='vumi.health', message_class=HeartBeatMessage) self._start_task()
def setup_transport(self): yield self.publish_status_starting() config = self.get_static_config() self.log.msg('Starting SMPP Transport for: %s' % (config.twisted_endpoint, )) default_prefix = '%s@%s' % (config.system_id, config.transport_name) redis_prefix = config.split_bind_prefix or default_prefix self.redis = (yield TxRedisManager.from_config( config.redis_manager)).sub_manager(redis_prefix) self.dr_processor = config.delivery_report_processor( self, config.delivery_report_processor_config) self.deliver_sm_processor = config.deliver_short_message_processor( self, config.deliver_short_message_processor_config) self.submit_sm_processor = config.submit_short_message_processor( self, config.submit_short_message_processor_config) self.disable_ack = config.disable_ack self.message_stash = SmppMessageDataStash(self.redis, config) self.service = self.start_service()
def setup_transport(self): warnings.warn( 'This SMPP implementation is deprecated. Please use the ' 'implementations available in vumi.transports.smpp.' 'smpp_transport instead.', category=DeprecationWarning) config = self.get_static_config() log.msg("Starting the SmppTransport for %s" % ( config.twisted_endpoint)) self.submit_sm_encoding = config.submit_sm_encoding self.submit_sm_data_coding = config.submit_sm_data_coding default_prefix = "%s@%s" % (config.system_id, config.transport_name) r_config = config.redis_manager r_prefix = config.split_bind_prefix or default_prefix redis = yield TxRedisManager.from_config(r_config) self.redis = redis.sub_manager(r_prefix) self.r_message_prefix = "message_json" self.throttled = False self.esme_callbacks = EsmeCallbacks( connect=self.esme_connected, disconnect=self.esme_disconnected, submit_sm_resp=self.submit_sm_resp, delivery_report=self.delivery_report, deliver_sm=self.deliver_sm) self._reconn_service = None if not hasattr(self, 'esme_client'): # start the Smpp transport (if we don't have one) self.factory = self.make_factory() self._reconn_service = ReconnectingClientService( config.twisted_endpoint, self.factory) self._reconn_service.startService()
def setup_worker(self): redis = yield TxRedisManager.from_config(self.config['redis_manager']) self.store = StatusStore(redis, ttl=None) yield self.unpause_connectors()