def change_task_status(self, ip, status, info={}): assert validate(ip) assert isinstance(info, dict) info.update({'create_dtm': datetime.datetime.now(), 'ip': ip}) dump_info = JSONEncoder().encode(info) if status == 'dispatch': c = tornadoredis.Client(connection_pool=self.pool) yield gen.Task(c.hset, 'task_dispatch', ip, dump_info) yield gen.Task(c.disconnect) elif status == 'checked': c = tornadoredis.Client(connection_pool=self.pool) if gen.Task(c.sismember, 'dispatch', ip): yield gen.Task(c.hset, 'task_checked', ip, dump_info) if info['PBL'] == '1': yield gen.Task(c.sadd, 'pbl_ips', ip) if info['SBL'] == '1': yield gen.Task(c.sadd, 'sbl_ips', ip) if info['XBL'] == '1': yield gen.Task(c.sadd, 'xbl_ips', ip) yield gen.Task(c.srem, 'dispatch', ip) yield gen.Task(c.disconnect) elif status == 'finished': c = tornadoredis.Client(connection_pool=self.pool) pipe = c.pipeline() pipe.hset('task_finished', ip, dump_info) pipe.srem('dispatch', ip) yield gen.Task(pipe.execute) yield gen.Task(c.disconnect)
def get(self): ''' Register the number of page views and return it on the page. Create NUMBER_OF_CLIENTS redis clients and connect them to the global connection pool. ''' indexes = range(1, NUMBER_OF_CLIENTS + 1) clients = (tornadoredis.Client(connection_pool=CONNECTION_POOL) for __ in indexes) # Register page views using %NUMBER_OF_CLIENTS% redis clients. # Run redis commands "simultaneously" to simulate a complex # application creating a number of Client instances. values = yield map( lambda c, n: tornado.gen.Task(self.incr_counter, c, n), clients, indexes) # Create a new client and get the redis server information # to display it on a demo page. c = tornadoredis.Client(connection_pool=CONNECTION_POOL) info = yield tornado.gen.Task(c.info) values = map(lambda n, v: (n, v), indexes, values) self.render("template.html", title="Connection Pool Demo", values=values, info=info)
def __init__(self, *args, **kwargs): super(Application, self).__init__(*args, **kwargs) self._mongodb_connection = orm.Connection self._mongodb_connection.connect(host=settings.mongodb.MONGODB_URL, db_name=settings.mongodb.DATABASE) self._fs = self._mongodb_connection.get_fs( db_name=settings.mongodb.FS_DATABASE) self._loader = ModelLoaderSingleton() self._async_redis = tornadoredis.Client( host=settings.redis.HOST, port=settings.redis.PORT, password=settings.redis.PASSWORD, selected_db=settings.redis.DATA) self._async_redis.connect() self._shark_redis = tornadoredis.Client( host=settings.redis.HOST, port=settings.redis.PORT, password=settings.redis.PASSWORD, selected_db=settings.redis.SHARK_DATA) self._shark_redis.connect() self._wukong_redis = self._async_redis self._loader.Cache(settings.redis.DATA_CACHE_KEY, self._async_redis) self._loader.Cache(settings.redis.SHARK_CACHE_KEY, self._shark_redis) self._loader.Cache(settings.redis.WUKONG_CACHE_KEY, self._async_redis) self._rq_connection = Redis(db=settings.redis.RQ) self._rq_queue = Queue(connection=self._rq_connection) self._loader.Cache(settings.redis.RQ_CACHE_KEY, self._rq_queue) HookFS(self._fs)
def get_redis(): if config['redis'].get('unix_socket_path') is not None: r = tornadoredis.Client( unix_socket_path=config['redis']['unix_socket_path']) else: r = tornadoredis.Client(host=config['redis']['host'], port=config['redis']['port']) r.connect() return r
def open(self, *args, **kwargs): # CONNECT AND GET THE ROOMS self.rooms = ['dummy'] self.client = tornadoredis.Client() self.pubclient = tornadoredis.Client() self.messages_published = [] self.client.connect() self.pubclient.connect() yield gen.Task(self.client.subscribe, self.rooms) self.client.listen(self.on_message_published)
def __init__(self, socket_path="/updates", websockets=True, subscriptions=None, redis_host="127.0.0.1", redis_port=6379, exit_gracefully=True, debug=False): """ RedisVisualization server. Servers updates from redis using websockets and also provide static files using Tornado. Inputs ------ @socket_path str : where should client ask for redis updates (default = '/updates') @websockets bool : use websockets ? (default = True) @subscriptions list<str> : what to subscribe to on Redis (default=['namespace_*', 'updates_*']) @redis_host str : what hostname is redis server on (default='127.0.0.1') @redis_port int : what port to listen to for redis server (default=6379) @exit_gracefully bool : capture SIGINT event & shut down server from ioloop (default=True) """ self.exit_gracefully = exit_gracefully if subscriptions is None: subscriptions = ["updates_*"] # 2. Start a connection pool to redis: pool = tornadoredis.ConnectionPool(host=redis_host, port=redis_port) self.clients = tornadoredis.Client(connection_pool=pool, password="") init_redis(tornadoredis.Client(connection_pool=pool, password="")) self.clients.connect() get_redis().connect() # make sure redis reports expiration and set events: try: self.clients.psubscribe( subscriptions, lambda msg: self.clients.listen(Connection.pubsub_message)) except tornadoredis.exceptions.ConnectionError: print(""" Could not connect to Redis. Start server with: > redis-server """) signal_handler(None, None) try_exit() if not websockets: Router = SockJSRouter(Connection, socket_path, dict(disabled_transports=['websocket'])) else: Router = SockJSRouter(Connection, socket_path) # 4. Creater router for http + sockets: self.App = tornado.web.Application( generate_routes(debug) + Router.urls)
def __init__(self): socket = get_redis_socket() if socket is not None: self._subscriber = tornadoredis.pubsub.SockJSSubscriber( tornadoredis.Client(unix_socket_path=socket, port=get_redis_port(), password=get_redis_password(), selected_db=get_redis_db())) else: self._subscriber = tornadoredis.pubsub.SockJSSubscriber( tornadoredis.Client(host=get_redis_host(), port=get_redis_port(), password=get_redis_password(), selected_db=get_redis_db()))
def attach(ioloop=None): global sub_hdr, blpop_hdr #消息队列客户端 blpop_hdr = tornadoredis.Client(host=config["host"], port=config["port"], selected_db=config["db"], io_loop=ioloop) #发布订阅客户端 sub_hdr = tornadoredis.Client(host=config["host"], port=config["port"], selected_db=config["db"], io_loop=ioloop) sub_hdr.connect() sub_listen()
def get_ips(self, name='', callback=None): if not bool(name): c = tornadoredis.Client(connection_pool=self.pool) customer = yield gen.Task(c.smembers, 'customer') data_list = [] for name in customer: ips = yield gen.Task(c.smembers, u'c_%s' % name) data_list.append({'name': name, 'data_list': {'ips': ips, 'num': len(ips)}}) yield gen.Task(c.disconnect) callback(data_list) else: c = tornadoredis.Client(connection_pool=self.pool) ips = yield gen.Task(c.smembers, u'c_%s' % name) callback([{'name': name, 'data_list': {'ips': ips, 'num': len(ips)}}])
def __init__(self, loop=None): self.loop = loop if not loop: self.loop = tornado.ioloop.IOLoop.instance() self.client = tornadoredis.Client()
def get_conn(self): """ 获取redis客户端链接 :return: """ resource = tornadoredis.Client(connection_pool=self.CONNECTION_POOL) raise tornado.gen.Return(resource)
def _connect_to_redis(self): self._redis_client = tornadoredis.Client(host=REDIS_SERVER, port=REDIS_PORT, selected_db=REDIS_DB) self._redis_client.connect()
def _new_client(self): client = tornadoredis.Client(io_loop=self.io_loop, port=self.test_port, selected_db=self.test_db) # client.connection.connect() # client.select(self.test_db) return client
class SessionBaseHandler(BaseHandler): send_client = redis.Client() send_client.connect() @property def channel(self): return self._channel @channel.setter def channel(self, value): self._channel = value def format_message(self, message): """ do user has send permission ? is message invalid ? return saveable message """ raise NotImplementedError def save_message(self, message): """ save message to database """ raise NotImplementedError def send_message(self, message): message = self.format_message(message) message = self.save_message(message) self.send_client.publish(self.channel, json_encode(message)) @gen.coroutine def listen(self): self.client = redis.Client() self.client.connect() yield gen.Task(self.client.subscribe, self.channel) self.client.listen(self._on_message) def _on_message(self, message): if message.kind == 'message': self.write_json(message.body) elif message.kind == 'disconnect': # Do not try to reconnect, just raise a error raise Exception('Redis server error') def on_close(self): """ Don't call this function in `RequestHandler` """ if self.client.subscribed: self.client.unsubscribe(self.channel) self.client.disconnect() # Do not call this function direct def write(self, message): if hasattr(self, 'write_message'): self.write_message(message) else: super(SessionBaseHandler, self).write(message) self.finish()
def GetSubscriber(): global __subscriber__ if not __subscriber__: __subscriber__ = SubscriptionSubscriber(tornadoredis.Client()) return __subscriber__
def handle(self, *args, **options): # connect to redis url = urlparse.urlparse(settings.REDIS_URL) pool = tornadoredis.ConnectionPool(host=url.hostname, port=url.port) redis_connection = tornadoredis.Client(connection_pool=pool, password=url.password) redis_connection.connect() sys.stdout.write("\nConnected to Redis ({}) \n".format(url)) redis_connection.psubscribe( "*", lambda message: redis_connection.listen( connections.MainConnection.pubsub_message)) # enable sockjs routing router = SockJSRouter( connections.MainConnection, '/chat', dict(disabled_transports=DISABLED_TRANSPORTS), ) app = web.Application( router.urls, "0.0.0.0", ) app.listen(IO_PORT) sys.stdout.write("\nApp listening at port {}\n".format(IO_PORT)) sys.stdout.flush() ioloop.IOLoop.instance().start()
def get(self): c = tornadoredis.Client() c.connect() pipe = c.pipeline() for i in xrange(1,5): keye = "entree" + str(i) keyd = "dessert" + str(i) keyb = "beverage" + str(i) keydtit = "dessert_title"+str(i) keyddesc = "dessert_desc" + str(i) keyetit = "entree_title" + str(i) keyedesc = "entree_desc" + str(i) keybtit = "beverage_title" + str(i) keybdesc = "beverage_desc" + str(i) entrees = {} desserts = {} beverages = {} dtitle = "" ddesc = "" etitle = "" edesc = "" btitle = "" bdesc = "" entrees[keyetit] = etitle entrees[keyedesc] = edesc desserts[keydtit] = dtitle desserts[keyddesc] = ddesc beverages[keybtit] = btitle beverages[keybdesc] = bdesc pipe.hmset(keye, entrees) pipe.hmset(keyd, desserts) pipe.hmset(keyb, beverages) yield tornado.gen.Task(pipe.execute) self.render("menu.html")
def get(self): self.set_header("Content-Type", "application/json") _cl = tornadoredis.Client() _initv = yield tornado.gen.Task(_cl.get, 'initv') data_json = tornado.escape.json_decode(_initv) self.write(data_json) self.finish()
def on_open(self, request): self.clients.add(self) self.redis = tornadoredis.Client(selected_db=6) self.redis.connect() self.redis.keys('*', self._keys_callback)
def make_app(): base_dir = path.dirname(path.abspath(__file__)) media_path = path.join(base_dir, "files") debug = True #redis = tornadoredis.ConnectionPool(max_connections=10, wait_for_available=True) redis = tornadoredis.Client() redis.connect() cache = DataManager() cache.redis = redis # cache.initialize() global_vars = dict(cache=cache) return Application( [ url(r"/", IndexHandler, global_vars, name="index"), url(r"/thread", ThreadHandler, global_vars, name="thread"), url(r"/ws", WsHandler, global_vars, name="ws"), url(r'/files/(.*)', StaticFileHandler, {'path': media_path}, name="files"), # url(r"/(?P<param1>.*)", HelloHandler, global_vars, name='home'), ], debug=debug, xsrf_cookies=False, template_path=path.join(base_dir, "templates"), static_path=path.join(base_dir, "static"), media_path=media_path, cookie_secret='secret', redis=redis, cache=cache, )
def __init__(self): self._subscriber = tornadoredis.pubsub.SockJSSubscriber( tornadoredis.Client(host=get_redis_host(), port=get_redis_port(), password=get_redis_pass(), selected_db=get_redis_db())) print get_redis_pass()
def setup_caching_store(): """ Create a global redis client and connect to the local instance """ global CACHING_STORE CACHING_STORE = tornadoredis.Client() CACHING_STORE.connect()
def get(self, *args, **kwargs): token = self.request.headers.get("token") c = tornadoredis.Client(connection_pool=db.REDIS_CONNECTION_POOL) user_id = yield gen.Task(c.zscore, 'user:token:list', token) c.disconnect() if user_id is None: self.set_status(401) self.finish( json.dumps({ 'code': "INVALID_ACCESS_TOKEN", 'message': u'无效的令牌' })) else: # 根据id order_item = yield db.POOL.execute( "SELECT order_id,total,user_id FROM `order`") order_item_id = order_item.fetchall() order_item.close() result_list = [] for key in order_item_id: order_item_item = yield db.POOL.execute( "SELECT food_id,count FROM `order_item` where order_id = '%s'" % key["order_id"]) order_item_items = order_item_item.fetchall() order_item_item.close() result = { 'id': key["order_id"], 'user_id': key["user_id"], 'total': key["total"], 'items': order_item_items } result_list.append(result) self.set_status(200) self.finish(json.dumps(result_list))
def redis_connect(redis_type='tornadoredis'): """ Established an asynchronous resi connection. """ redistogo_url = os.getenv('REDISTOGO_URL', None) if redistogo_url == None: REDIS_HOST = 'localhost' REDIS_PORT = 6379 REDIS_PWD = None REDIS_USER = None else: redis_url = redistogo_url redis_url = redis_url.split('redis://')[1] redis_url = redis_url.split('/')[0] REDIS_USER, redis_url = redis_url.split(':', 1) REDIS_PWD, redis_url = redis_url.split('@', 1) REDIS_HOST, REDIS_PORT = redis_url.split(':', 1) if redis_type == 'redis': client = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PWD) client.ping() elif redis == 'tornadoredis': client = tornadoredis.Client() client.connect() else: return None return client
def get_async_conn(id): if id in _ASYNC_CONNS_: return _ASYNC_CONNS_[id] _ASYNC_CONNS_[id] = conn = tornadoredis.Client(io_loop=_IO_LOOP_, **configs['redis'][id]) conn.connect() return conn
def get_handler_class(self): redis_conn = tornadoredis.Client() redis_conn.connect() rate_limit = RateLimit(redis_conn, namespace=gen_random_string()) class Handler(RequestHandler): def user(self): return self.request.body @rate_limit.limit('5/2s') def get(self): self.finish("decorator") @coroutine def post(self): with (yield rate_limit.limit('user:5/2s', user=self.user).cm()): self.finish("contextmanager") def write_error(self, status_code, **kwargs): self.set_status(200) if 'exc_info' in kwargs: if issubclass(kwargs['exc_info'][0], RateLimitExceeded): self.finish("rate_limit_exceeded") else: self.finish("this shouldn't happen") return Handler
def post(self): c = tornadoredis.Client(connection_pool=CONNECTION_POOL) url = self.get_argument("url") if not util.validate_url(url): self.write("not a valid url") self.finish() return key = util.gen_cache_key(const.CACHE_KEY_PREFIX.REVERSE_URL, url) short_id = redis_short_id = yield tornado.gen.Task(c.get, key) while not redis_short_id: short_id = util.gen_short_id() key = util.gen_cache_key(const.CACHE_KEY_PREFIX.SHORT_ID, short_id) ret = yield tornado.gen.Task(c.get, key) if ret: continue yield tornado.gen.Task(c.set, key, url) key = util.gen_cache_key(const.CACHE_KEY_PREFIX.REVERSE_URL, url) yield tornado.gen.Task(c.set, key, short_id) break self.write(urlparse.urljoin(config.SITE_URL, short_id)) self.finish()
def __init__(self, *args, **kwargs): self.closed_channels = None self.parsable_prefix = 'p' super(MessagesHandler, self).__init__(*args, **kwargs) self.id = id(self) self.ip = None from chat import global_redis self.async_redis_publisher = global_redis.async_redis_publisher self.sync_redis = global_redis.sync_redis self.channels = [] self.call_receiver_channel = None self._logger = None self.async_redis = tornadoredis.Client() self.patch_tornadoredis() self.pre_process_message = { Actions.GET_MESSAGES: self.process_get_messages, Actions.SEND_MESSAGE: self.process_send_message, Actions.CALL: self.process_call, Actions.CREATE_DIRECT_CHANNEL: self.create_user_channel, Actions.DELETE_ROOM: self.delete_channel, Actions.EDIT_MESSAGE: self.edit_message, Actions.CREATE_ROOM_CHANNEL: self.create_new_room, Actions.INVITE_USER: self.invite_user, } self.post_process_message = { Actions.CREATE_DIRECT_CHANNEL: self.send_client_new_channel, Actions.CREATE_ROOM_CHANNEL: self.send_client_new_channel, Actions.DELETE_ROOM: self.send_client_delete_channel, Actions.INVITE_USER: self.send_client_new_channel, Actions.CALL: self.set_opponent_call_channel }
def listen(self): self.client = tornadoredis.Client() self.client.connect() yield tornado.gen.Task(self.client.subscribe, 'rv') self.client.listen(self.on_message)
def __init__(self, persist=True): super(Carbon, self).__init__() self.metrics = set() self.redis = tornadoredis.Client() # Maybe some parameters self.redis.connect() if persist: self.persist = Subprocess( ["python", "-m", "whirlwind.tornado.carbon.persist"])