def configure_processors(self): self.data_processor = RedisBackendPeriodicProcessor(self, io_loop=self.io_loop, config=self.config) self.data_processor.start()
class RedisBackend(BaseBackend): def __init__(self, application, io_loop, config, storage, tasks, data_threshold=500): self.io_loop = io_loop self.config = config self.storage = storage self.client = None self.tasks = tasks self.data_processing_threshold = data_threshold self.current_in_progress = 0 self.data_processor = None self.application = application self.working = True def get_redis_client(self): """Get redis client instance """ client = Client( #connection_pool=connection_pool, host=self.backend_settings.get('HOST', 'localhost'), port=int(self.backend_settings.get('PORT', 6379)), io_loop=self.io_loop, password=self.backend_settings.get('PASSWORD', None), selected_db=int(self.backend_settings.get('DB', 0)), reconnect_callback=self.listen) return client def configure_client(self): """Make redis client instance """ self.client = self.get_redis_client() return self.client def configure_processors(self): self.data_processor = RedisBackendPeriodicProcessor(self, io_loop=self.io_loop, config=self.config) self.data_processor.start() @classmethod def setup_backend(cls, application, io_loop, config, storage, tasks): """Setup data handler :param io_loo: :class:`tornado.ioloop.IOLoop` object :param confi: :class:`gottwall.config.Config` object :param storage: :class:`gottwall.storage.Storage` object """ backend = cls(application, io_loop, config, storage, tasks) backend.configure_client() backend.configure_processors() backend.listen() return backend @gen.engine def listen(self): """Listen socket :param client: redis client """ self.client.connect() yield tornado.gen.Task( self.client.psubscribe, "{0}:*".format(self.backend_settings.get('CHANNEL', 'gottwall'))) self.client.listen(self.callback) def callback(self, message): if message.body == 1: # handshake return try: project, public_key, private_key = self.parse_channel(message.channel) except ValueError: logger.warn("Invalid channel credentails") return except Exception, e: logger.warn(e) data = self.parse_data(message.body.strip(), project) message_type = data.get('action', None) if message_type == 'notification': # Load data from key to deque for project self.load_buckets(project) elif message_type in ['incr', 'decr']: self.process_data(project, data.get('action'), self.parsed_data_to_list(data)) return True