def change_task_status(self, ip, status, info={}):
assert validate(ip)
assert isinstance(info, dict)
info.update({'create_dtm': datetime.datetime.now(), 'ip': ip})
dump_info = JSONEncoder().encode(info)
if status == 'dispatch':
c = tornadoredis.Client(connection_pool=self.pool)
yield gen.Task(c.hset, 'task_dispatch', ip, dump_info)
yield gen.Task(c.disconnect)
elif status == 'checked':
c = tornadoredis.Client(connection_pool=self.pool)
if gen.Task(c.sismember, 'dispatch', ip):
yield gen.Task(c.hset, 'task_checked', ip, dump_info)
if info['PBL'] == '1':
yield gen.Task(c.sadd, 'pbl_ips', ip)
if info['SBL'] == '1':
yield gen.Task(c.sadd, 'sbl_ips', ip)
if info['XBL'] == '1':
yield gen.Task(c.sadd, 'xbl_ips', ip)
yield gen.Task(c.srem, 'dispatch', ip)
yield gen.Task(c.disconnect)
elif status == 'finished':
c = tornadoredis.Client(connection_pool=self.pool)
pipe = c.pipeline()
pipe.hset('task_finished', ip, dump_info)
pipe.srem('dispatch', ip)
yield gen.Task(pipe.execute)
yield gen.Task(c.disconnect)
def get(self):
'''
Register the number of page views and return it on the page.
Create NUMBER_OF_CLIENTS redis clients and connect them
to the global connection pool.
'''
indexes = range(1, NUMBER_OF_CLIENTS + 1)
clients = (tornadoredis.Client(connection_pool=CONNECTION_POOL)
for __ in indexes)
# Register page views using %NUMBER_OF_CLIENTS% redis clients.
# Run redis commands "simultaneously" to simulate a complex
# application creating a number of Client instances.
values = yield map(
lambda c, n: tornado.gen.Task(self.incr_counter, c, n), clients,
indexes)
# Create a new client and get the redis server information
# to display it on a demo page.
c = tornadoredis.Client(connection_pool=CONNECTION_POOL)
info = yield tornado.gen.Task(c.info)
values = map(lambda n, v: (n, v), indexes, values)
self.render("template.html",
title="Connection Pool Demo",
values=values,
info=info)
def __init__(self, *args, **kwargs):
super(Application, self).__init__(*args, **kwargs)
self._mongodb_connection = orm.Connection
self._mongodb_connection.connect(host=settings.mongodb.MONGODB_URL,
db_name=settings.mongodb.DATABASE)
self._fs = self._mongodb_connection.get_fs(
db_name=settings.mongodb.FS_DATABASE)
self._loader = ModelLoaderSingleton()
self._async_redis = tornadoredis.Client(
host=settings.redis.HOST,
port=settings.redis.PORT,
password=settings.redis.PASSWORD,
selected_db=settings.redis.DATA)
self._async_redis.connect()
self._shark_redis = tornadoredis.Client(
host=settings.redis.HOST,
port=settings.redis.PORT,
password=settings.redis.PASSWORD,
selected_db=settings.redis.SHARK_DATA)
self._shark_redis.connect()
self._wukong_redis = self._async_redis
self._loader.Cache(settings.redis.DATA_CACHE_KEY, self._async_redis)
self._loader.Cache(settings.redis.SHARK_CACHE_KEY, self._shark_redis)
self._loader.Cache(settings.redis.WUKONG_CACHE_KEY, self._async_redis)
self._rq_connection = Redis(db=settings.redis.RQ)
self._rq_queue = Queue(connection=self._rq_connection)
self._loader.Cache(settings.redis.RQ_CACHE_KEY, self._rq_queue)
HookFS(self._fs)
def get_redis():
if config['redis'].get('unix_socket_path') is not None:
r = tornadoredis.Client(
unix_socket_path=config['redis']['unix_socket_path'])
else:
r = tornadoredis.Client(host=config['redis']['host'],
port=config['redis']['port'])
r.connect()
return r
def open(self, *args, **kwargs):
# CONNECT AND GET THE ROOMS
self.rooms = ['dummy']
self.client = tornadoredis.Client()
self.pubclient = tornadoredis.Client()
self.messages_published = []
self.client.connect()
self.pubclient.connect()
yield gen.Task(self.client.subscribe, self.rooms)
self.client.listen(self.on_message_published)
def __init__(self,
socket_path="/updates",
websockets=True,
subscriptions=None,
redis_host="127.0.0.1",
redis_port=6379,
exit_gracefully=True,
debug=False):
"""
RedisVisualization server. Servers updates from redis using websockets
and also provide static files using Tornado.
Inputs
------
@socket_path str : where should client ask for redis updates (default = '/updates')
@websockets bool : use websockets ? (default = True)
@subscriptions list<str> : what to subscribe to on Redis (default=['namespace_*', 'updates_*'])
@redis_host str : what hostname is redis server on (default='127.0.0.1')
@redis_port int : what port to listen to for redis server (default=6379)
@exit_gracefully bool : capture SIGINT event & shut down server from ioloop (default=True)
"""
self.exit_gracefully = exit_gracefully
if subscriptions is None:
subscriptions = ["updates_*"]
# 2. Start a connection pool to redis:
pool = tornadoredis.ConnectionPool(host=redis_host, port=redis_port)
self.clients = tornadoredis.Client(connection_pool=pool, password="")
init_redis(tornadoredis.Client(connection_pool=pool, password=""))
self.clients.connect()
get_redis().connect()
# make sure redis reports expiration and set events:
try:
self.clients.psubscribe(
subscriptions,
lambda msg: self.clients.listen(Connection.pubsub_message))
except tornadoredis.exceptions.ConnectionError:
print("""
Could not connect to Redis. Start server with:
> redis-server
""")
signal_handler(None, None)
try_exit()
if not websockets:
Router = SockJSRouter(Connection, socket_path,
dict(disabled_transports=['websocket']))
else:
Router = SockJSRouter(Connection, socket_path)
# 4. Creater router for http + sockets:
self.App = tornado.web.Application(
generate_routes(debug) + Router.urls)
def __init__(self):
socket = get_redis_socket()
if socket is not None:
self._subscriber = tornadoredis.pubsub.SockJSSubscriber(
tornadoredis.Client(unix_socket_path=socket,
port=get_redis_port(),
password=get_redis_password(),
selected_db=get_redis_db()))
else:
self._subscriber = tornadoredis.pubsub.SockJSSubscriber(
tornadoredis.Client(host=get_redis_host(),
port=get_redis_port(),
password=get_redis_password(),
selected_db=get_redis_db()))
def attach(ioloop=None):
global sub_hdr, blpop_hdr
#消息队列客户端
blpop_hdr = tornadoredis.Client(host=config["host"],
port=config["port"],
selected_db=config["db"],
io_loop=ioloop)
#发布订阅客户端
sub_hdr = tornadoredis.Client(host=config["host"],
port=config["port"],
selected_db=config["db"],
io_loop=ioloop)
sub_hdr.connect()
sub_listen()
def get_ips(self, name='', callback=None):
if not bool(name):
c = tornadoredis.Client(connection_pool=self.pool)
customer = yield gen.Task(c.smembers, 'customer')
data_list = []
for name in customer:
ips = yield gen.Task(c.smembers, u'c_%s' % name)
data_list.append({'name': name, 'data_list': {'ips': ips, 'num': len(ips)}})
yield gen.Task(c.disconnect)
callback(data_list)
else:
c = tornadoredis.Client(connection_pool=self.pool)
ips = yield gen.Task(c.smembers, u'c_%s' % name)
callback([{'name': name, 'data_list': {'ips': ips, 'num': len(ips)}}])
def __init__(self, loop=None):
self.loop = loop
if not loop:
self.loop = tornado.ioloop.IOLoop.instance()
self.client = tornadoredis.Client()
def get_conn(self):
"""
获取redis客户端链接
:return:
"""
resource = tornadoredis.Client(connection_pool=self.CONNECTION_POOL)
raise tornado.gen.Return(resource)
def _connect_to_redis(self):
self._redis_client = tornadoredis.Client(host=REDIS_SERVER,
port=REDIS_PORT,
selected_db=REDIS_DB)
self._redis_client.connect()
def _new_client(self):
client = tornadoredis.Client(io_loop=self.io_loop,
port=self.test_port,
selected_db=self.test_db)
# client.connection.connect()
# client.select(self.test_db)
return client
class SessionBaseHandler(BaseHandler):
send_client = redis.Client()
send_client.connect()
@property
def channel(self):
return self._channel
@channel.setter
def channel(self, value):
self._channel = value
def format_message(self, message):
"""
do user has send permission ?
is message invalid ?
return saveable message
"""
raise NotImplementedError
def save_message(self, message):
"""
save message to database
"""
raise NotImplementedError
def send_message(self, message):
message = self.format_message(message)
message = self.save_message(message)
self.send_client.publish(self.channel, json_encode(message))
@gen.coroutine
def listen(self):
self.client = redis.Client()
self.client.connect()
yield gen.Task(self.client.subscribe, self.channel)
self.client.listen(self._on_message)
def _on_message(self, message):
if message.kind == 'message':
self.write_json(message.body)
elif message.kind == 'disconnect':
# Do not try to reconnect, just raise a error
raise Exception('Redis server error')
def on_close(self):
"""
Don't call this function in `RequestHandler`
"""
if self.client.subscribed:
self.client.unsubscribe(self.channel)
self.client.disconnect()
# Do not call this function direct
def write(self, message):
if hasattr(self, 'write_message'):
self.write_message(message)
else:
super(SessionBaseHandler, self).write(message)
self.finish()
def GetSubscriber():
global __subscriber__
if not __subscriber__:
__subscriber__ = SubscriptionSubscriber(tornadoredis.Client())
return __subscriber__
def handle(self, *args, **options):
# connect to redis
url = urlparse.urlparse(settings.REDIS_URL)
pool = tornadoredis.ConnectionPool(host=url.hostname, port=url.port)
redis_connection = tornadoredis.Client(connection_pool=pool,
password=url.password)
redis_connection.connect()
sys.stdout.write("\nConnected to Redis ({}) \n".format(url))
redis_connection.psubscribe(
"*", lambda message: redis_connection.listen(
connections.MainConnection.pubsub_message))
# enable sockjs routing
router = SockJSRouter(
connections.MainConnection,
'/chat',
dict(disabled_transports=DISABLED_TRANSPORTS),
)
app = web.Application(
router.urls,
"0.0.0.0",
)
app.listen(IO_PORT)
sys.stdout.write("\nApp listening at port {}\n".format(IO_PORT))
sys.stdout.flush()
ioloop.IOLoop.instance().start()
def get(self):
c = tornadoredis.Client()
c.connect()
pipe = c.pipeline()
for i in xrange(1,5):
keye = "entree" + str(i)
keyd = "dessert" + str(i)
keyb = "beverage" + str(i)
keydtit = "dessert_title"+str(i)
keyddesc = "dessert_desc" + str(i)
keyetit = "entree_title" + str(i)
keyedesc = "entree_desc" + str(i)
keybtit = "beverage_title" + str(i)
keybdesc = "beverage_desc" + str(i)
entrees = {}
desserts = {}
beverages = {}
dtitle = ""
ddesc = ""
etitle = ""
edesc = ""
btitle = ""
bdesc = ""
entrees[keyetit] = etitle
entrees[keyedesc] = edesc
desserts[keydtit] = dtitle
desserts[keyddesc] = ddesc
beverages[keybtit] = btitle
beverages[keybdesc] = bdesc
pipe.hmset(keye, entrees)
pipe.hmset(keyd, desserts)
pipe.hmset(keyb, beverages)
yield tornado.gen.Task(pipe.execute)
self.render("menu.html")
def get(self):
self.set_header("Content-Type", "application/json")
_cl = tornadoredis.Client()
_initv = yield tornado.gen.Task(_cl.get, 'initv')
data_json = tornado.escape.json_decode(_initv)
self.write(data_json)
self.finish()
def on_open(self, request):
self.clients.add(self)
self.redis = tornadoredis.Client(selected_db=6)
self.redis.connect()
self.redis.keys('*', self._keys_callback)
def make_app():
base_dir = path.dirname(path.abspath(__file__))
media_path = path.join(base_dir, "files")
debug = True
#redis = tornadoredis.ConnectionPool(max_connections=10, wait_for_available=True)
redis = tornadoredis.Client()
redis.connect()
cache = DataManager()
cache.redis = redis
# cache.initialize()
global_vars = dict(cache=cache)
return Application(
[
url(r"/", IndexHandler, global_vars, name="index"),
url(r"/thread", ThreadHandler, global_vars, name="thread"),
url(r"/ws", WsHandler, global_vars, name="ws"),
url(r'/files/(.*)',
StaticFileHandler, {'path': media_path},
name="files"),
# url(r"/(?P<param1>.*)", HelloHandler, global_vars, name='home'),
],
debug=debug,
xsrf_cookies=False,
template_path=path.join(base_dir, "templates"),
static_path=path.join(base_dir, "static"),
media_path=media_path,
cookie_secret='secret',
redis=redis,
cache=cache,
)
def __init__(self):
self._subscriber = tornadoredis.pubsub.SockJSSubscriber(
tornadoredis.Client(host=get_redis_host(),
port=get_redis_port(),
password=get_redis_pass(),
selected_db=get_redis_db()))
print get_redis_pass()
def setup_caching_store():
"""
Create a global redis client and connect to the local instance
"""
global CACHING_STORE
CACHING_STORE = tornadoredis.Client()
CACHING_STORE.connect()
def get(self, *args, **kwargs):
token = self.request.headers.get("token")
c = tornadoredis.Client(connection_pool=db.REDIS_CONNECTION_POOL)
user_id = yield gen.Task(c.zscore, 'user:token:list', token)
c.disconnect()
if user_id is None:
self.set_status(401)
self.finish(
json.dumps({
'code': "INVALID_ACCESS_TOKEN",
'message': u'无效的令牌'
}))
else:
# 根据id
order_item = yield db.POOL.execute(
"SELECT order_id,total,user_id FROM `order`")
order_item_id = order_item.fetchall()
order_item.close()
result_list = []
for key in order_item_id:
order_item_item = yield db.POOL.execute(
"SELECT food_id,count FROM `order_item` where order_id = '%s'"
% key["order_id"])
order_item_items = order_item_item.fetchall()
order_item_item.close()
result = {
'id': key["order_id"],
'user_id': key["user_id"],
'total': key["total"],
'items': order_item_items
}
result_list.append(result)
self.set_status(200)
self.finish(json.dumps(result_list))
def redis_connect(redis_type='tornadoredis'):
"""
Established an asynchronous resi connection.
"""
redistogo_url = os.getenv('REDISTOGO_URL', None)
if redistogo_url == None:
REDIS_HOST = 'localhost'
REDIS_PORT = 6379
REDIS_PWD = None
REDIS_USER = None
else:
redis_url = redistogo_url
redis_url = redis_url.split('redis://')[1]
redis_url = redis_url.split('/')[0]
REDIS_USER, redis_url = redis_url.split(':', 1)
REDIS_PWD, redis_url = redis_url.split('@', 1)
REDIS_HOST, REDIS_PORT = redis_url.split(':', 1)
if redis_type == 'redis':
client = redis.StrictRedis(host=REDIS_HOST,
port=REDIS_PORT,
password=REDIS_PWD)
client.ping()
elif redis == 'tornadoredis':
client = tornadoredis.Client()
client.connect()
else:
return None
return client
def get_async_conn(id):
if id in _ASYNC_CONNS_:
return _ASYNC_CONNS_[id]
_ASYNC_CONNS_[id] = conn = tornadoredis.Client(io_loop=_IO_LOOP_,
**configs['redis'][id])
conn.connect()
return conn
def get_handler_class(self):
redis_conn = tornadoredis.Client()
redis_conn.connect()
rate_limit = RateLimit(redis_conn, namespace=gen_random_string())
class Handler(RequestHandler):
def user(self):
return self.request.body
@rate_limit.limit('5/2s')
def get(self):
self.finish("decorator")
@coroutine
def post(self):
with (yield rate_limit.limit('user:5/2s', user=self.user).cm()):
self.finish("contextmanager")
def write_error(self, status_code, **kwargs):
self.set_status(200)
if 'exc_info' in kwargs:
if issubclass(kwargs['exc_info'][0], RateLimitExceeded):
self.finish("rate_limit_exceeded")
else:
self.finish("this shouldn't happen")
return Handler
def post(self):
c = tornadoredis.Client(connection_pool=CONNECTION_POOL)
url = self.get_argument("url")
if not util.validate_url(url):
self.write("not a valid url")
self.finish()
return
key = util.gen_cache_key(const.CACHE_KEY_PREFIX.REVERSE_URL, url)
short_id = redis_short_id = yield tornado.gen.Task(c.get, key)
while not redis_short_id:
short_id = util.gen_short_id()
key = util.gen_cache_key(const.CACHE_KEY_PREFIX.SHORT_ID, short_id)
ret = yield tornado.gen.Task(c.get, key)
if ret:
continue
yield tornado.gen.Task(c.set, key, url)
key = util.gen_cache_key(const.CACHE_KEY_PREFIX.REVERSE_URL, url)
yield tornado.gen.Task(c.set, key, short_id)
break
self.write(urlparse.urljoin(config.SITE_URL, short_id))
self.finish()
def __init__(self, *args, **kwargs):
self.closed_channels = None
self.parsable_prefix = 'p'
super(MessagesHandler, self).__init__(*args, **kwargs)
self.id = id(self)
self.ip = None
from chat import global_redis
self.async_redis_publisher = global_redis.async_redis_publisher
self.sync_redis = global_redis.sync_redis
self.channels = []
self.call_receiver_channel = None
self._logger = None
self.async_redis = tornadoredis.Client()
self.patch_tornadoredis()
self.pre_process_message = {
Actions.GET_MESSAGES: self.process_get_messages,
Actions.SEND_MESSAGE: self.process_send_message,
Actions.CALL: self.process_call,
Actions.CREATE_DIRECT_CHANNEL: self.create_user_channel,
Actions.DELETE_ROOM: self.delete_channel,
Actions.EDIT_MESSAGE: self.edit_message,
Actions.CREATE_ROOM_CHANNEL: self.create_new_room,
Actions.INVITE_USER: self.invite_user,
}
self.post_process_message = {
Actions.CREATE_DIRECT_CHANNEL: self.send_client_new_channel,
Actions.CREATE_ROOM_CHANNEL: self.send_client_new_channel,
Actions.DELETE_ROOM: self.send_client_delete_channel,
Actions.INVITE_USER: self.send_client_new_channel,
Actions.CALL: self.set_opponent_call_channel
}
def listen(self):
self.client = tornadoredis.Client()
self.client.connect()
yield tornado.gen.Task(self.client.subscribe, 'rv')
self.client.listen(self.on_message)
def __init__(self, persist=True):
super(Carbon, self).__init__()
self.metrics = set()
self.redis = tornadoredis.Client() # Maybe some parameters
self.redis.connect()
if persist:
self.persist = Subprocess(
["python", "-m", "whirlwind.tornado.carbon.persist"])
