def init():
app = web.Application()
loop = asyncio.get_event_loop()
handler = app.make_handler()
f = loop.create_server(handler, SERVE_ON, SERVE_ON_PORT)
srv = loop.run_until_complete(f)
add_routes(app)
app['redis_pool'] = loop.run_until_complete(aioredis.create_pool(REDIS))
app['redis_subscribe_pool'] = loop.run_until_complete(aioredis.create_pool(REDIS))
print('serving on', srv.sockets[0].getsockname())
try:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
srv.close()
loop.run_until_complete(app['redis_pool'].clear())
loop.run_until_complete(app['redis_subscribe_pool'].clear())
loop.run_until_complete(srv.wait_closed())
loop.run_until_complete(app.shutdown())
loop.run_until_complete(handler.finish_connections(60.0))
loop.run_until_complete(app.cleanup())
loop.close()
def connectRedis(self):
if len(self.setAddressList) > 0:
for var_address in self.setAddressList:
# TODO encoding=utf-8
try:
objRedisPool = yield from aioredis.create_pool(address=var_address, loop=self.objAioLoopObj,
db=self.iDb,
password=self.strPwd, minsize=self.iMinPoolSize,
maxsize=self.iMaxPoolSize,encoding=self.strEncodeing)
self.objAioRedisDict[(var_address[0]+":"+str(var_address[1]))] = objRedisPool
except Exception as e:
print("aioredis init failed address{} port{} db{} pwd{}".format(var_address[0],var_address[1],self.iDb, self.strPwd))
raise e
else:
#TODO encoding=utf-8
try:
self.objAioRedis = yield from aioredis.create_pool(address=self.setAddress,loop=self.objAioLoopObj,db=self.iDb,
password=self.strPwd,minsize=self.iMinPoolSize,maxsize=self.iMaxPoolSize,encoding=self.strEncodeing)
except Exception as e:
print("aioredis init failed address{} port{} db{} pwd{}".format(self.setAddress[0],self.setAddress[1],
self.iDb,self.strPwd))
raise e
def init_redis_pool(self, loop=None):
"""
Init redis connection pool
:param loop: ioloop
"""
redis_addr = options.redis_socket if options.use_redis_socket \
else (options.redis_host, options.redis_port)
# TODO validate redis connection params
redis_psw = options.redis_psw
if redis_psw == "":
redis_psw = None
redis_db = options.redis_db
if redis_db == -1:
redis_db = None
self.log_info("Init Redis connection pool... ")
self.log_info(f"ADDR={redis_addr} DB={redis_db}",
grep_label=INIT_REDIS_LABEL)
self.log_info(
f"MIN_POOL_SIZE={options.redis_min_con} "
f"MAX_POOL_SIZE={options.redis_max_con}",
grep_label=INIT_REDIS_LABEL)
self.redis_connection_pool = loop.run_until_complete(
aioredis.create_pool(redis_addr,
password=redis_psw,
db=redis_db,
minsize=options.redis_min_con,
maxsize=options.redis_max_con))
def pubsub():
pool = yield from aioredis.create_pool(
('localhost', 6379),
minsize=5, maxsize=10)
@asyncio.coroutine
def reader(channel):
while (yield from channel.wait_message()):
msg = yield from channel.get(encoding='utf-8')
# ... process message ...
print("message in {}: {}".format(channel.name, msg))
if msg == STOPWORD:
return
with (yield from pool) as redis:
channel, = yield from redis.subscribe('channel:1')
yield from reader(channel) # wait for reader to complete
yield from redis.unsubscribe('channel:1')
# Explicit redis usage
redis = yield from pool.acquire()
try:
channel, = yield from redis.subscribe('channel:1')
yield from reader(channel) # wait for reader to complete
yield from redis.unsubscribe('channel:1')
finally:
pool.release(redis)
yield from pool.clear() # closing all open connections
def init(loop):
adminMainController = AdminMainController()
engine = yield from create_engine(**config['db'])
with (yield from engine) as connection:
adminUserController = AdminUserController(
adminMainController, users, connection)
redis = yield from aioredis.create_pool(('localhost', 6379))
storage = RedisStorage(redis)
app = web.Application(middlewares=[session_middleware(storage)])
aiohttp_jinja2.setup(app,
loader=jinja2.FileSystemLoader('templates'))
app.router.add_route('GET', '/', main_page_view)
app.router.add_route('GET', '/admin', adminMainController.get)
app.router.add_route(
'GET', '/admin/{table}', adminMainController.get_table)
app.router.add_route(
'POST', '/admin/{table}', adminMainController.create_record)
app.router.add_static('/dist', 'dist')
server = yield from loop.create_server(app.make_handler(),
'127.0.0.1', 8080)
return server
def initialize(self):
if self._pool is None:
self._pool = yield aioredis.create_pool(self._addr, **self._settings)
app_log.info(r'MCachePool initialized')
def test_no_yield_from(self):
pool = yield from create_pool(
('localhost', self.redis_port), loop=self.loop)
with self.assertRaises(RuntimeError):
with pool:
pass
def lookup(config, request):
global pool
global lock
if request.client is None:
return None
metadata = None
# standard check/lock/check pattern to ensure only one thread creates a connection pool
if pool is None:
with (yield from lock):
if pool is None:
pool = yield from aioredis.create_pool(
(config.redis_host, config.redis_port),
minsize=2,
maxsize=20)
# Call the eval script to lookup IP and retrieve instance data.
# Could probably optimize this by storing the script server-side
# during initial pool creation.
with (yield from pool) as redis:
pickle_data = yield from redis.eval(KEY_SCRIPT,
args=[KEY_IP,
str(request.client)])
if pickle_data is not None:
metadata = pickle.loads(pickle_data)
return metadata
def __init__(
self,
config: RedisConfig,
*,
loop: asyncio.BaseEventLoop = None,
expiration: int = 300,
key_pattern: Tuple[AvailableKeys] = DEFAULT_KEY_PATTERN,
encrypt_key=True,
):
"""
:param loop:
:type loop:
"""
BaseCache.__init__(self, config.expiration)
_loop = loop or asyncio.get_event_loop()
self._redis_pool = _loop.run_until_complete(
aioredis.create_pool(
(config.host, config.port), db=config.db, password=config.password
)
)
self.key_prefix = config.key_prefix
super().__init__(
expiration=expiration, key_pattern=key_pattern, encrypt_key=encrypt_key
)
def __init__(self):
client_id = settings.DISCORD_APP_ID
intents = discord.Intents.default()
intents.members = True
super().__init__(
command_prefix="!",
description=description,
intents=intents,
)
self.redis = None
self.redis = self.loop.run_until_complete(aioredis.create_pool(getattr(settings, "BROKER_URL", "redis://localhost:6379/0"), minsize=5, maxsize=10))
print('redis pool started', self.redis)
self.client_id = client_id
self.session = aiohttp.ClientSession(loop=self.loop)
self.tasks = []
self.message_connection = Connection(getattr(settings, "BROKER_URL", 'redis://localhost:6379/0'))
queues = []
for que in queue_keys:
queues.append(Queue(que))
self.message_consumer = Consumer(self.message_connection, queues, callbacks=[self.on_queue_message], accept=['json'])
django.setup()
for hook in hooks.get_hooks("discord_cogs_hook"):
for cog in hook():
try:
self.load_extension(cog)
except Exception as e:
print(f"Failed to load cog {cog}", file=sys.stderr)
traceback.print_exc()
def pubsub():
pool = yield from aioredis.create_pool(('localhost', 6379),
minsize=5,
maxsize=10)
@asyncio.coroutine
def reader(channel):
while (yield from channel.wait_message()):
msg = yield from channel.get(encoding='utf-8')
# ... process message ...
print("message in {}: {}".format(channel.name, msg))
if msg == STOPWORD:
return
with (yield from pool) as conn:
raw_result = yield from conn.execute_pubsub('subscribe', 'channel:1')
print('raw result:', raw_result)
channel = conn.pubsub_channels['channel:1']
yield from reader(channel) # wait for reader to complete
yield from conn.execute_pubsub('unsubscribe', 'channel:1')
# Explicit connection usage
conn = yield from pool.acquire()
try:
raw_result = yield from conn.execute_pubsub('subscribe', 'channel:1')
print('raw result:', raw_result)
channel = conn.pubsub_channels['channel:1']
yield from reader(channel) # wait for reader to complete
yield from conn.execute_pubsub('unsubscribe', 'channel:1')
finally:
pool.release(conn)
pool.close()
yield from pool.wait_closed() # closing all open connections
def test_clear(self):
pool = yield from create_pool(
('localhost', self.redis_port), loop=self.loop)
self._assert_defaults(pool)
yield from pool.clear()
self.assertEqual(pool.freesize, 0)
def __init__(self, cfg, is_debug):
"""
:param cfg: 配置项
"""
self.cfg = cfg
self.token = cfg.get('general', 'token')
self.protocols = {}
self.conn_ip = (cfg.get('general',
'conn_ip'), cfg.getint('general',
'listen_port'))
self.server_address = (cfg.get('general', 'listen_ip'),
cfg.getint('general', 'listen_port'))
self.expiration_time = cfg.getint('general', 'expiration_time')
self.loop = uvloop.new_event_loop()
if is_debug:
self.loop.set_debug(True)
task = self.loop.create_task(
aioredis.create_pool('redis://{}:{}'.format(
cfg.get('redis', 'host'), cfg.getint('redis', 'port')),
minsize=cfg.getint('redis', 'min_size'),
maxsize=cfg.getint('redis', 'max_size'),
loop=self.loop))
task.add_done_callback(self.set_redis)
factory = self.loop.create_server(lambda: Protocol(self),
*self.server_address)
self.server = self.loop.run_until_complete(factory)
def get_pool(cls):
if not cls._pool:
with (yield from cls._lock):
if not cls._pool:
cls._pool = yield from aioredis.create_pool((cls._host, cls._port), minsize=cls._minsize,
maxsize=cls._maxsize)
return (yield from cls._pool)
def init(loop):
app = web.Application(loop=loop)
# -------------------------------------------------------------------------
app.router.add_route(
'GET', r'/{thread_id:\d+}/{user_id:\d+}/', PrivateMessageHandler()
)
# -------------------------------------------------------------------------
app['pg_pool'] = yield from aiopg.create_pool(dsn)
app['redis_pool'] = yield from aioredis.create_pool(redis_address)
# -------------------------------------------------------------------------
srv = yield from loop.create_server(
app.make_handler(), SERVER['HOST'], SERVER['PORT']
)
print(
"Server started at http://{0}:{1}".format(
SERVER['HOST'], SERVER['PORT']
)
)
return srv
def get_pool(self):
if not self._pool:
with (yield from self._lock):
if not self._pool:
self._pool = yield from aioredis.create_pool((self._host, self._port), minsize=self._minsize,
maxsize=self._maxsize)
return (yield from self._pool)
def __init__(self, redis_ip: str = "127.0.0.1", redis_port: int = 6379,
side: int = 0, thread_count: int = 2):
"""
Create a new task runner.
:param side: The side. 0 for client, 1 for server. Automatically set.
:param redis_ip: The IP address for the Redis server.
:param redis_port: The port for the Redis server.
"""
if PY33:
self._loop = asyncio.get_event_loop()
self._r_ip = redis_ip
self._r_port = redis_port
# Create a redis connection
if PY33:
self.redis_conn = self._loop.run_until_complete(aioredis.create_pool((self._r_ip, self._r_port)))
self._blocking_redis = redis.Redis(host=self._r_ip, port=self._r_port)
else:
self._blocking_redis = redis.Redis(host=self._r_ip, port=self._r_port)
self.thread_count = thread_count
self.tasks = {}
# Change side.
if os.environ.pop("KOHLRABI_SERVER", "0") == "1":
# Force server-side
self.side = 1
else:
self.side = side
if self.side == 1:
self._logger_output()
else:
logger.debug("Kohlrabi client object loaded and connected")
def create_pool(self, loop=None):
self.pool = yield from create_pool('redis://localhost:6379',
password='******',
minsize=50,
maxsize=120,
db=7,
loop=loop)
def main(crawler_conf, redis_conf, mysql_conf, execute_dir):
# 注：要想开启 asyncio 的日志，必须 import 的顺序是 asyncio 在前。
log_dir_path = os.path.join(execute_dir, 'log')
if not os.path.exists(log_dir_path):
os.makedirs(log_dir_path)
log_file_path = os.path.join(execute_dir, 'log', 'stackoverflow.log')
log_handler = logging.FileHandler(log_file_path)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
log_handler.setFormatter(formatter)
logger = logging.getLogger('asyncio')
logger.setLevel(logging.INFO)
logger.addHandler(log_handler)
# 初始化事件循环
loop = uvloop.new_event_loop()
asyncio.set_event_loop(loop)
max_concurrent_sem = asyncio.Semaphore(value=crawler_conf['consumer_sem'],
loop=loop)
# 初始化全局 aiohttp.ClientSession 和 redis pool
initialize_tasks = list()
initialize_tasks.append(create_global_session(loop))
initialize_tasks.append(
aioredis.create_pool((redis_conf['host'], redis_conf['port']),
db=redis_conf['db'],
minsize=redis_conf['pool_minsize'],
maxsize=redis_conf['pool_maxsize'],
loop=loop))
results = loop.run_until_complete(asyncio.gather(*initialize_tasks))
aiohttp_session = results[0]
redis_pool = results[1]
tasks = list()
for _ in range(crawler_conf['consumer_num']):
tasks.append(consumer(aiohttp_session, redis_pool, max_concurrent_sem))
# 收到 kill 信号之后，主程序可以做 finally 操作。
def signal_term_handler(sig, frame):
sys.exit(0)
# TODO handle kill 信号，将失效的单词放回队列中。
signal.signal(signal.SIGTERM, signal_term_handler)
try:
loop.run_until_complete(asyncio.gather(*tasks))
except KeyboardInterrupt:
pass
finally:
loop.run_until_complete(
asyncio.ensure_future(close_session(aiohttp_session), loop=loop))
loop.run_until_complete(
asyncio.ensure_future(close_redis_pool(redis_pool), loop=loop))
time.sleep(5)
loop.close()
def config_db(registry, settings):
# DB 0 cauth
# DB 1 tauth
# DB 2 token
# authorization_codes
# TOKEN::scope : client_id
db_pool_cauths = yield from aioredis.create_pool(
(settings['redis.host'], settings['redis.port']),
db=0,
minsize=5,
maxsize=10)
# service_tokens
# TOKEN : CLIENT_ID
db_pool_tauth = yield from aioredis.create_pool(
(settings['redis.host'], settings['redis.port']),
db=1,
minsize=5,
maxsize=10)
# User tokens
# TOKEN :
db_pool_token = yield from aioredis.create_pool(
(settings['redis.host'], settings['redis.port']),
db=2,
minsize=5,
maxsize=10)
# db_pool_cauths = redis.ConnectionPool(
# host=settings['redis.host'], port=settings['redis.port'], db=0)
# db_pool_tauth = redis.ConnectionPool(
# host=settings['redis.host'], port=settings['redis.port'], db=1)
# db_pool_token = redis.ConnectionPool(
# host=settings['redis.host'], port=settings['redis.port'], db=2)
# db_conn_cauths = redis.Redis(connection_pool=db_pool_cauths)
# db_conn_tauth = redis.Redis(connection_pool=db_pool_tauth)
# db_conn_token = redis.Redis(connection_pool=db_pool_token)
# db_conn_scopes = {}
# db_conn_client_id = {}
registry['settings']['db_cauths'] = db_pool_cauths
registry['settings']['db_tauths'] = db_pool_tauth
registry['settings']['db_token'] = db_pool_token
def connect(cls, host, port, minsize=5, maxsize=10, loop=asyncio.get_event_loop()):
"""
Setup a connection pool
:param host: Redis host
:param port: Redis port
:param loop: Event loop
"""
cls._pool = yield from aioredis.create_pool((host, port), minsize=minsize, maxsize=maxsize, loop=loop)
def test_select_db(self):
pool = yield from create_pool(
('localhost', self.redis_port),
loop=self.loop)
yield from pool.select(1)
with (yield from pool) as redis:
self.assertEqual(redis.db, 1)
def init(loop):
# add aiohttp_debugtoolbar middleware to you application
app = web.Application(loop=loop)
extra_panels = []
if 'aiopg' in sys.modules:
extra_panels.append(RequestPgDebugPanel)
if 'aioredis' in sys.modules:
extra_panels.append(RequestRedisDebugPanel)
# install aiohttp_debugtoolbar
aiohttp_debugtoolbar.setup(app,
extra_panels=extra_panels,
extra_templates=str(PATH_PARENT / 'extra_tpl'))
template = """
<html>
<head>
<title>{{ title }}</title>
</head>
<body>
<h1>{{ text }}</h1>
<p>
<a href="{{ app.router['exc_example'].url_for() }}">
Exception example</a>
</p>
</body>
</html>
"""
# install jinja2 templates
loader = jinja2.DictLoader({'index.html': template})
aiohttp_jinja2.setup(app, loader=loader)
# init routes for index page, and page with error
app.router.add_route('GET', '/', basic_handler, name='index')
app.router.add_route('GET', '/exc', exception_handler, name='exc_example')
if 'aiopg' in sys.modules:
# create connection to the database
dsn = 'host={host} dbname={db} user={user} password={passw} '.format(
db='postgres', user='******', passw='1', host='localhost')
app['db'] = yield from aiopg.create_pool(dsn,
loop=loop,
minsize=1,
maxsize=2)
# Correct PostgreSQL shutdown
app.on_cleanup.append(close_pg)
if 'aioredis' in sys.modules:
# create redis pool
app['redis'] = yield from create_pool(('127.0.0.1', '6379'))
# Correct Redis shutdown
app.on_cleanup.append(close_redis)
handler = app.make_handler()
srv = yield from loop.create_server(handler, '127.0.0.1', 9000)
print("Server started at http://127.0.0.1:9000")
return srv, handler
def go():
pool = yield from aioredis.create_pool(
('localhost', 6379),
minsize=5, maxsize=10, db=1,
loop=loop)
with (yield from pool) as redis: # high-level redis API instance
yield from redis.set('my-key', 'value')
print((yield from redis.get('my-key')))
pool.clear() # closing all open connections
def init(loop):
# add aiohttp_debugtoolbar middleware to you application
app = web.Application(loop=loop)
extra_panels = []
if 'aiopg' in sys.modules:
extra_panels.append(RequestPgDebugPanel)
if 'aioredis' in sys.modules:
extra_panels.append(RequestRedisDebugPanel)
# install aiohttp_debugtoolbar
aiohttp_debugtoolbar.setup(
app,
extra_panels=extra_panels,
extra_templates=str(PATH_PARENT / 'extra_tpl'))
template = """
<html>
<head>
<title>{{ title }}</title>
</head>
<body>
<h1>{{ text }}</h1>
<p>
<a href="{{ app.router['exc_example'].url_for() }}">
Exception example</a>
</p>
</body>
</html>
"""
# install jinja2 templates
loader = jinja2.DictLoader({'index.html': template})
aiohttp_jinja2.setup(app, loader=loader)
# init routes for index page, and page with error
app.router.add_route('GET', '/', basic_handler, name='index')
app.router.add_route('GET', '/exc', exception_handler, name='exc_example')
if 'aiopg' in sys.modules:
# create connection to the database
dsn = 'host={host} dbname={db} user={user} password={passw} '.format(
db='postgres', user='******', passw='1', host='localhost')
app['db'] = yield from aiopg.create_pool(
dsn, loop=loop, minsize=1, maxsize=2)
# Correct PostgreSQL shutdown
app.on_cleanup.append(close_pg)
if 'aioredis' in sys.modules:
# create redis pool
app['redis'] = yield from create_pool(('127.0.0.1', '6379'))
# Correct Redis shutdown
app.on_cleanup.append(close_redis)
handler = app.make_handler()
srv = yield from loop.create_server(handler, '127.0.0.1', 9000)
print("Server started at http://127.0.0.1:9000")
return srv, handler
def go():
pool = yield from aioredis.create_pool(
('localhost', 6379),
minsize=5, maxsize=10)
with (yield from pool) as redis: # high-level redis API instance
yield from redis.set('my-key', 'value')
val = yield from redis.get('my-key')
print('raw value:', val)
yield from pool.clear() # closing all open connections
def get_pool(cls):
if not cls._pool:
with (yield from cls._lock):
if not cls._pool:
cls._pool = yield from aioredis.create_pool(
(cls._host, cls._port),
minsize=cls._minsize,
maxsize=cls._maxsize)
return (yield from cls._pool)
def get_pool(self):
if not self._pool:
with (yield from self._lock):
if not self._pool:
self._pool = yield from aioredis.create_pool(
(self._host, self._port),
minsize=self._minsize,
maxsize=self._maxsize)
return (yield from self._pool)
def go():
pool = yield from aioredis.create_pool(
('localhost', 6379),
minsize=5, maxsize=10)
with (yield from pool) as conn: # low-level redis connection
yield from conn.execute('set', 'my-key', 'value')
val = yield from conn.execute('get', 'my-key')
print('raw value:', val)
pool.close()
yield from pool.wait_closed() # closing all open connections
def go():
pool = yield from aioredis.create_pool(
('localhost', 6379),
minsize=5, maxsize=10)
with (yield from pool) as redis: # high-level redis API instance
yield from redis.set('my-key', 'value')
val = yield from redis.get('my-key')
print('raw value:', val)
pool.close()
yield from pool.wait_closed() # closing all open connections
def test_release_bad_connection(self):
pool = yield from create_pool(
('localhost', self.redis_port),
loop=self.loop)
yield from pool.acquire()
other_conn = yield from create_redis(
('localhost', self.redis_port),
loop=self.loop)
with self.assertRaises(AssertionError):
pool.release(other_conn)
def init(loop):
redis = yield from aioredis.create_pool(('localhost', 6379), loop=loop)
storage = aiohttp_session.redis_storage.RedisStorage(redis)
session_middleware = aiohttp_session.session_middleware(storage)
app = aiohttp.web.Application(middlewares=[session_middleware])
app.router.add_route('GET', '/', handler)
srv = yield from loop.create_server(
app.make_handler(), '0.0.0.0', 8081)
return srv
def __await__(self):
self._pool = yield from aioredis.create_pool(
**self._settings).__await__()
Utils.log.info(
f"Redis {self._settings[r'address']} initialized: {self._pool.size}/{self._pool.maxsize}"
)
return self
def open(self):
# Create Redis connection
# if self.connection is None:
# self.connection = yield from asyncio_redis.Connection.create(host=self.host, port=self.port)
# if self.connection is None:
# self.connection = yield from aioredis.create_connection((self.host, self.port))
if RedisClass.connection is None:
RedisClass.connection = yield from aioredis.create_pool((self.host, self.port), minsize=10, maxsize=1000)
print('redis opended.')
def test_release_closed(self):
pool = yield from create_pool(
('localhost', self.redis_port),
minsize=1, loop=self.loop)
self.assertEqual(pool.size, 1)
self.assertEqual(pool.freesize, 1)
with (yield from pool) as redis:
redis.close()
self.assertEqual(pool.size, 0)
self.assertEqual(pool.freesize, 0)
def test_response_decoding(self):
pool = yield from create_pool(
('localhost', self.redis_port),
encoding='utf-8', loop=self.loop)
self.assertEqual(pool.encoding, 'utf-8')
with (yield from pool) as redis:
yield from redis.set('key', 'value')
with (yield from pool) as redis:
res = yield from redis.get('key')
self.assertEqual(res, 'value')
def __init__(self,
config: RedisConfig,
*,
loop: asyncio.BaseEventLoop = None):
"""
:param loop:
:type loop:
"""
BaseCache.__init__(self, config.expiration)
_loop = loop or asyncio.get_event_loop()
if config.url is None:
self._redis_pool = _loop.run_until_complete(
aioredis.create_pool((config.host, config.port),
db=config.db,
password=config.password))
else:
self._redis_pool = _loop.run_until_complete(
aioredis.create_pool(config.url))
self.key_prefix = config.key_prefix
def test_simple_command(self):
pool = yield from create_pool(
('localhost', self.redis_port),
minsize=10, loop=self.loop)
with (yield from pool) as conn:
msg = yield from conn.echo('hello')
self.assertEqual(msg, b'hello')
self.assertEqual(pool.size, 10)
self.assertEqual(pool.freesize, 9)
self.assertEqual(pool.size, 10)
self.assertEqual(pool.freesize, 10)
def init(config_object: Optional[Any] = None, **kwargs) -> None:
"""
初始化 miraibot
此函数必须在代码的开始处调用
否则，got_bot() 函数将返回 None 且无法正常工作
:param config_object: 包含框架所需的配置的对象
"""
import logging
global app, loop, bcc, redis
if config_object is None:
from . import default_config as config_object
config_dict = {
k: v
for k, v in config_object.__dict__.items()
if k.isupper() and not k.startswith('_')
}
for i in ('HOST', 'PORT', 'AUTHKEY', 'QQ'):
if i not in config_dict:
raise ValueError(f'配置文件中缺失 HOST 或 PORT 或 AUTHKEY 或 QQ: {i}')
if config_dict['DEBUG']:
logger.setLevel(logging.DEBUG)
debug = True
else:
logger.setLevel(logging.INFO)
debug = False
loop = asyncio.get_event_loop()
bcc = Broadcast(loop=loop, debug_flag=debug)
loop.create_task(schedule.run_pending())
app = miraibot(
config_dict=config_dict,
logger=logger,
broadcast=bcc,
debug=debug,
**kwargs
)
if aioredis is not None and config_dict["REDIS"]:
redis = asyncio.get_event_loop().run_until_complete(
aioredis.create_pool(
f"redis://{config_dict['REDIS_HOST']}:{config_dict['REDIS_PORT']}", # noqa
db=config_dict["REDIS_DB"], password=config_dict["REDIS_PASSWORD"], # noqa
minsize=config_dict["REDIS_MINSIZE"], maxsize=config_dict["REDIS_MAXSIZE"], # noqa
loop = loop # noqa
)
)
def test_crappy_multiexec(self):
pool = yield from create_pool(
('localhost', self.redis_port),
encoding='utf-8', loop=self.loop,
minsize=1, maxsize=1)
with (yield from pool) as redis:
yield from redis.set('abc', 'def')
yield from redis.connection.execute('multi')
yield from redis.set('abc', 'fgh')
self.assertTrue(redis.closed)
with (yield from pool) as redis:
value = yield from redis.get('abc')
self.assertEquals(value, 'def')
def test_create_constraints(self):
pool = yield from create_pool(
('localhost', self.redis_port),
minsize=1, maxsize=1, loop=self.loop)
self.assertEqual(pool.size, 1)
self.assertEqual(pool.freesize, 1)
with (yield from pool):
self.assertEqual(pool.size, 1)
self.assertEqual(pool.freesize, 0)
with self.assertRaises(asyncio.TimeoutError):
yield from asyncio.wait_for(pool.acquire(),
timeout=0.2,
loop=self.loop)
def create_server(self, method, path, handler=None, max_age=None):
self.redis = yield from aioredis.create_pool(("localhost", 6379), minsize=5, maxsize=10, loop=self.loop)
self.addCleanup(self.redis.clear)
middleware = session_middleware(RedisStorage(self.redis, max_age=max_age))
app = web.Application(middlewares=[middleware], loop=self.loop)
if handler:
app.router.add_route(method, path, handler)
port = self.find_unused_port()
handler = app.make_handler()
srv = yield from self.loop.create_server(handler, "127.0.0.1", port)
url = "http://127.0.0.1:{}".format(port) + path
self.srv = srv
self.handler = handler
return app, srv, url
def test():
pool = yield from create_pool(
('localhost', self.redis_port),
minsize=1, db=0,
loop=self.loop)
db = 0
while True:
db = (db + 1) & 1
res = yield from asyncio.gather(pool.select(db),
pool.acquire(),
loop=self.loop)
conn = res[1]
self.assertEqual(pool.db, db)
pool.release(conn)
if conn.db == db:
break
def test_create_no_minsize(self):
pool = yield from create_pool(
('localhost', self.redis_port),
minsize=0, maxsize=1, loop=self.loop)
self.assertEqual(pool.size, 0)
self.assertEqual(pool.freesize, 0)
with (yield from pool):
self.assertEqual(pool.size, 1)
self.assertEqual(pool.freesize, 0)
with (yield from pool):
self.assertEqual(pool.size, 2)
self.assertEqual(pool.freesize, 0)
self.assertEqual(pool.size, 1)
self.assertEqual(pool.freesize, 1)
def __init__(self,
config: RedisConfig,
*,
loop: asyncio.BaseEventLoop = None):
"""
:param loop:
:type loop:
"""
BaseCache.__init__(self, config.expiration)
_loop = loop or asyncio.get_event_loop()
self._redis_pool = _loop.run_until_complete(aioredis.create_pool((config.host, config.port),
db=config.db,
password=config.password))
self.key_prefix = config.key_prefix
def connect(cls,
host,
port,
minsize=5,
maxsize=10,
loop=asyncio.get_event_loop()):
"""
Setup a connection pool
:param host: Redis host
:param port: Redis port
:param loop: Event loop
"""
cls._pool = yield from aioredis.create_pool((host, port),
minsize=minsize,
maxsize=maxsize,
loop=loop)
def test_aioredis():
import aioredis
pool = yield from aioredis.create_pool(
('localhost', 6379),
minsize=2, maxsize=2)
base = [0]
@ring.func.aioredis(pool, 'ring-test', 5)
@asyncio.coroutine
def cached_function(a, b):
return str(base[0] + a * 100 + b).encode()
yield from common_test(cached_function, base)
pool.close()
yield from pool.wait_closed()
def create_server(self, method, path, handler, max_age=None):
self.redis = yield from aioredis.create_pool(('localhost', 6379),
minsize=5,
maxsize=10,
loop=self.loop)
self.addCleanup(self.redis.clear)
middleware = session_middleware(
RedisStorage(self.redis, max_age=max_age))
app = web.Application(middlewares=[middleware], loop=self.loop)
app.router.add_route(method, path, handler)
port = self.find_unused_port()
handler = app.make_handler()
srv = yield from self.loop.create_server(handler, '127.0.0.1', port)
url = "http://127.0.0.1:{}".format(port) + path
self.srv = srv
self.handler = handler
return app, srv, url
def init(loop):
redis_pool = yield from create_pool(('localhost', 6379))
dbengine = yield from create_engine(user='******',
password='******',
database='aiohttp_security',
host='127.0.0.1')
app = web.Application(loop=loop)
setup_session(app, RedisStorage(redis_pool))
setup_security(app, SessionIdentityPolicy(),
DBAuthorizationPolicy(dbengine))
web_handlers = Web()
yield from web_handlers.configure(app)
handler = app.make_handler()
srv = yield from loop.create_server(handler, '127.0.0.1', 8080)
print("Server started at http://127.0.0.1:8080")
return srv, app, handler
def __init__(self, *, config):
super().__init__(command_prefix=self.get_prefix,
pm_help=None,
description=DESCRIPTION)
# Load config
self.config = config
# Connect to mongo
self._mongo = AsyncIOMotorClient(self.config['database'])
self.db = self._mongo['destroyer']
# Redis
_redis_future = aioredis.create_pool(('localhost', 6379), db=11)
self.redis = self.loop.run_until_complete(_redis_future)
log.info('Loading cogs.')
self.load_cogs()
def app_factory(loop):
global logger
redis_conf = getattr(settings, 'REDIS', {})
logger.info('Create Redis connections pool')
redis_pool = yield from aioredis.create_pool(
(redis_conf.get('HOST', '127.0.0.1'), redis_conf.get('PORT', 6379)),
db=redis_conf.get('DB', 0),
password=redis_conf.get('PASSWORD', None),
loop=loop)
logger.info('Create Application object')
app = web.Application(
middlewares=[session_middleware(RedisStorage(redis_pool))])
app.logger = logger
app.redis_pool = redis_pool
return app
def init(loop):
redis_pool = yield from create_pool(('localhost', 6379))
dbengine = yield from create_engine(user='******',
password='******',
database='aiohttp_security',
host='127.0.0.1')
app = web.Application(loop=loop)
setup_session(app, RedisStorage(redis_pool))
setup_security(app,
SessionIdentityPolicy(),
DBAuthorizationPolicy(dbengine))
web_handlers = Web()
yield from web_handlers.configure(app)
handler = app.make_handler()
srv = yield from loop.create_server(handler, '127.0.0.1', 8080)
print("Server started at http://127.0.0.1:8080")
return srv, app, handler
def app_factory(loop):
global logger
redis_conf = getattr(settings, 'REDIS', {})
logger.info('Create Redis connections pool')
redis_pool = yield from aioredis.create_pool(
(redis_conf.get('HOST', '127.0.0.1'), redis_conf.get('PORT', 6379)),
db=redis_conf.get('DB', 0),
password=redis_conf.get('PASSWORD', None),
loop=loop
)
logger.info('Create Application object')
app = web.Application(middlewares=[session_middleware(RedisStorage(redis_pool))])
app.logger = logger
app.redis_pool = redis_pool
return app
def run_server(host, port):
loop = asyncio.get_event_loop()
redis_pool = loop.run_until_complete(create_pool(('localhost', 6379)))
client = motor_asyncio.AsyncIOMotorClient('localhost', 27017)
# Initiate main application
app = web.Application(loop=loop, logger=server_logger)
setup_routes(app, 'main')
setup_static(app)
aiohttp_jinja2.setup(app, loader=jinja2.FileSystemLoader('templates/'))
# Initiate sub-applications
admin = web.Application(loop=loop, logger=server_logger)
setup_routes(admin, 'admin')
aiohttp_jinja2.setup(admin, loader=jinja2.FileSystemLoader('templates/'))
setup_session(
admin,
RedisStorage(redis_pool,
max_age=COOKIE_AGE,
cookie_name=COOKIE_AUTH_NAME))
setup_security(admin, SessionIdentityPolicy(), DBAuthorizationPolicy())
app.add_subapp('/admin/', admin)
api = web.Application(loop=loop, logger=server_logger)
setup_routes(api, 'api')
app.add_subapp('/api/', api)
# Session and security
setup_session(
app,
RedisStorage(redis_pool,
max_age=COOKIE_AGE,
cookie_name=COOKIE_AUTH_NAME))
setup_security(app, SessionIdentityPolicy(), DBAuthorizationPolicy())
# Run server
web.run_app(app=app,
host=host,
port=port,
access_log=access_logger,
access_log_format=
'%a %l %u %t "%r" %s %b "%{Referrer}i" "%{User-Agent}i"')
def __init__(self):
client_id = settings.DISCORD_APP_ID
super().__init__(
command_prefix="!",
description=description,
)
self.redis = None
self.redis = self.loop.run_until_complete(
aioredis.create_pool(("localhost", 6379), minsize=5, maxsize=10))
print('redis pool started', self.redis)
self.client_id = client_id
self.session = aiohttp.ClientSession(loop=self.loop)
django.setup()
for cog in initial_cogs:
try:
self.load_extension("aadiscordbot.{0}".format(cog))
except Exception as e:
print(f"Failed to load cog {cog}", file=sys.stderr)
traceback.print_exc()
def __init__(self,
redis_ip: str = "127.0.0.1",
redis_port: int = 6379,
side: int = 0,
thread_count: int = 2):
"""
Create a new task runner.
:param side: The side. 0 for client, 1 for server. Automatically set.
:param redis_ip: The IP address for the Redis server.
:param redis_port: The port for the Redis server.
"""
if PY33:
self._loop = asyncio.get_event_loop()
self._r_ip = redis_ip
self._r_port = redis_port
# Create a redis connection
if PY33:
self.redis_conn = self._loop.run_until_complete(
aioredis.create_pool((self._r_ip, self._r_port)))
self._blocking_redis = redis.Redis(host=self._r_ip,
port=self._r_port)
else:
self._blocking_redis = redis.Redis(host=self._r_ip,
port=self._r_port)
self.thread_count = thread_count
self.tasks = {}
# Change side.
if os.environ.pop("KOHLRABI_SERVER", "0") == "1":
# Force server-side
self.side = 1
else:
self.side = side
if self.side == 1:
self._logger_output()
else:
logger.debug("Kohlrabi client object loaded and connected")
def start(loop, api_host='127.0.0.1', api_port=9876):
app = web.Application(loop=loop)
app['mock_db'] = {}
redis_address = (config('ECHOD_REDIS_HOST', default='127.0.0.1'),
config('ECHOD_REDIS_PORT', default=6379))
redis_db = int(config('ECHOD_REDIS_DB', default=0))
redis_pool = yield from aioredis.create_pool(redis_address, db=redis_db,
minsize=5, maxsize=10,
encoding='utf-8', loop=loop)
app['redis_pool'] = redis_pool
# Mock
app.router.add_route('GET', '/mocks/', get_mock)
app.router.add_route('PUT', '/mocks/', put_mock)
# Proxies
app.router.add_route('GET', '/proxies/', get_proxy)
app.router.add_route('PUT', '/proxies/', put_proxy)
# Callbacks
app.router.add_route('*', '/callbacks/{app}/{queue}/', callback)
app.router.add_route('GET', '/callbacks/_all/{app}/{queue}/', all_callback)
app.router.add_route('GET', '/callbacks/_first/{app}/{queue}/',
first_callback)
app.router.add_route('GET', '/callbacks/_last/{app}/{queue}/',
last_callback)
app.router.add_route('GET', '/callbacks/_clean/{app}/{queue}/',
clean_callback)
# Health
app.router.add_route('GET', '/health/', health)
handler = app.make_handler()
server = yield from loop.create_server(handler, api_host, api_port)
address = server.sockets[0].getsockname()
log.info('API started at http://{}:{}/'.format(*address))
return server, handler, redis_pool
def test_change_db_errors(self):
pool = yield from create_pool(
('localhost', self.redis_port),
minsize=1, db=0,
loop=self.loop)
with self.assertRaises(TypeError):
yield from pool.select(None)
self.assertEqual(pool.db, 0)
with (yield from pool):
pass
self.assertEqual(pool.size, 1)
self.assertEqual(pool.freesize, 1)
with self.assertRaises(TypeError):
yield from pool.select(None)
self.assertEqual(pool.db, 0)
with self.assertRaises(ValueError):
yield from pool.select(-1)
self.assertEqual(pool.db, 0)
with self.assertRaises(ReplyError):
yield from pool.select(100000)
self.assertEqual(pool.db, 0)
def f(*args, **kw):
redis = yield from aioredis.create_pool(*args, **kw)
_closable(redis)
return redis
