def main():
loop = asyncio.get_event_loop()
loop.create_task(example_producer())
ExampleWebsocketServer(
redis=loop.run_until_complete(create_redis_pool(REDIS_ADDRESS)),
subscriber=loop.run_until_complete(create_redis_pool(REDIS_ADDRESS)),
read_timeout=30,
keep_alive_timeout=120,
).listen(host='localhost',
port=8000,
channel_names=('example_channel_1', 'example_channel_2'),
loop=loop)
def __init__(self, loop: asyncio.BaseEventLoop):
super().__init__(loop)
# Socket dumper queue dumper task.
self.extra_tasks.append(self.process_queues)
self.ingest_queue = queue.Queue()
self.sockets_ingesting = {}
self.active_rooms = set()
self.redis = aioredis.create_redis_pool(
os.environ.get("REDIS_URL", "redis://localhost"),
minsize=5,
maxsize=69, # hehe 69
loop=loop,
)
# Setup HTTP session
conn = aiohttp.TCPConnector(limit=0) # Uncap the max HTTP connections.
self.session = aiohttp.ClientSession(
loop=loop,
headers={
"User-Agent": "RPAN socket dumper sponsored by u/nepeat"
},
connector=conn,
)
async def bootstrap(app):
settings = app[SETTINGS]
redis_addr = settings.get('db.redis', 'localhost:6379')
host, port = redis_addr.strip().split(':')
app[DB_REDIS] = await asyncio.wait_for(
aioredis.create_redis_pool((host, int(port)), minsize=0, maxsize=5, loop=app.loop), 3, loop=app.loop)
app[HTTP_SESSION] = await ClientSession().__aenter__()
async def connect(self) -> None:
"""Connect to Redis by creating the connections pool."""
log.debug("Creating Redis connections pool.")
# Decide if we want to use `fakeredis` or an actual Redis server. The
# option to use `fakeredis.aioredis` is provided to aid with running the
# application in a development environment and to aid with writing
# unittests.
if self._use_fakeredis:
# Only import fakeredis when required. This ensures that it's not a
# required dependency if someone's not planning on using it.
try:
import fakeredis.aioredis
except ImportError:
raise FakeRedisNotInstalled(
"RedisSession was configured to use `fakeredis`, but it is not installed. "
"Either install `fakeredis` manually or install `async-rediscache` using "
"`pip install async-rediscache[fakeredis]` to enable support."
)
kwargs = dict(self._session_kwargs)
# the `address` and `password` kwargs are not supported by redis
kwargs.pop("address", None)
kwargs.pop("password", None)
pool_constructor = fakeredis.aioredis.create_redis_pool(**kwargs)
else:
pool_constructor = aioredis.create_redis_pool(
**self._session_kwargs)
self._pool = await pool_constructor
async def prepare_conn(self):
addresses = conf.REDIS['addresses']
password = conf.REDIS.get('password')
self.redis_pool = await asyncio.gather(*[
aioredis.create_redis_pool(addresses[i], minsize=1, maxsize=1)
for i in range(self.max)
])
logger.info('RedisClient init done')
async def get_redis_client():
import asyncio
import aioredis
redis = RedisDataBase()
redis.client = await asyncio.wait_for(
aioredis.create_redis_pool("redis://localhost:6379"), 3)
return redis.client
async def connect_to_redis():
logger.info("Connecting to Redis")
# try:
redis.client = await asyncio.wait_for(
aioredis.create_redis_pool(settings.REDIS_CONNECTION), 3)
# except asyncio.TimeoutError:
# logger.error('connect_to_redis timeout!')
logger.info("Connected to Redis")
def aioredis_pool():
if sys.version_info >= (3, 5):
import aioredis
pool_coroutine = aioredis.create_redis_pool(
('localhost', 6379), minsize=2, maxsize=2)
return pool_coroutine, ring.aioredis
else:
pytest.skip()
async def connect(self, loop=None):
create_pool = aioredis.create_redis_pool(
(self.redis_host, self.redis_port),
password=self.redis_password,
minsize=5,
maxsize=10,
loop=loop)
self.pool = await create_pool
async def get_redis(host: str, timeout: float = 120):
"""
blocks until connected to redis
"""
return (await asyncio.wait({
create_task(
aioredis.create_redis_pool(f"redis://{host}", timeout=timeout))
}))[0].pop().result()
def __init__(self, host='127.0.0.1', port='6379', db=0, password=None):
self.jobs = {}
self.queue_name = 'macul:default:task'
self.redis = aioredis.create_redis_pool(
f'redis://{host}:{port}',
db=db,
password=password,
)
async def on_started():
global game_store
# TODO: azure docker compose doesn't support depends so timeout
logger.info("connecting to redis")
task = asyncio.create_task(
aioredis.create_redis_pool("redis://game_store", timeout=120))
game_store = (await asyncio.wait({task}))[0].pop().result()
assert await game_store.ping()
logger.info("connected to redis")
async def connect_db():
asyncio._set_running_loop(asyncio.get_event_loop())
redis = await trio_asyncio.run_asyncio(
aioredis.create_redis_pool(
REDIS_URI,
password=REDIS_PASSWORD,
encoding='utf-8',
))
app.db = Database(redis)
def init_cache(conf):
global redis, redis_lru
host = conf['redis-host']
port = conf['redis-port']
rdb = conf['redis-db']
logger.warn('initializing cache(redis): {} {} {}'.format(host, port, rdb))
# don't know why yet but must run like this before running aiohttp
loop = asyncio.get_event_loop()
redis = loop.run_until_complete(
aioredis.create_redis_pool((host, port), db=rdb, maxsize=50))
host = conf['redis-lru-host']
port = conf['redis-lru-port']
rdb = conf['redis-lru-db']
logger.warn('initializing cache(redis-lru): {} {} {}'.format(
host, port, rdb))
# don't know why yet but must run like this before running aiohttp
loop = asyncio.get_event_loop()
redis_lru = loop.run_until_complete(
aioredis.create_redis_pool((host, port), db=rdb, maxsize=50))
async def get_redis_client(poolsize=None):
redis_client = None
try:
host = TannerConfig.get('REDIS', 'host')
port = TannerConfig.get('REDIS', 'port')
if poolsize is None:
poolsize = TannerConfig.get('REDIS', 'poolsize')
timeout = TannerConfig.get('REDIS', 'timeout')
redis_client = await asyncio.wait_for(aioredis.create_redis_pool(
(host, int(port)), maxsize=int(poolsize)), timeout=int(timeout))
except asyncio.TimeoutError as timeout_error:
LOGGER.exception('Problem with redis connection. Please, check your redis server. %s', timeout_error)
exit()
return redis_client
def __init__(self, handlers):
logging.info("Server starting...")
AsyncIOMainLoop().install()
self.loop = asyncio.get_event_loop()
self.router = Router(handlers)
self.app = Application(self.router.routes, **cfg.app.tornado_settings)
self.app.loop = self.loop
if hasattr(cfg, "db"):
from microservice.models import connection
self.app.objects = peewee_async.Manager(connection, loop=self.loop)
else:
self.app.objects = None
self.app.sentry_client = AsyncSentryClient(cfg.app.sentry_url, **cfg.app.sentry_client_kwargs)
try:
import aioredis
if hasattr(cfg.redis, "socket") and cfg.redis.socket:
self.app.redis_connection = self.loop.run_until_complete(aioredis.create_redis_pool(cfg.redis.socket))
else:
self.app.redis_connection = self.loop.run_until_complete(aioredis.create_redis_pool((cfg.redis.host, cfg.redis.port)))
except (ImportError, OSError, AttributeError):
self.app.redis_connection = None
def __init__(self, name):
# self.redis = asyncio.run()
self.crawler = MonitorCrawler()
self.name = name
self.loop = asyncio.new_event_loop()
self.redis = self.loop.run_until_complete(
aioredis.create_redis_pool('redis://localhost'))
key_names = ['init', 'tmp', 'last', 'speed', 'score']
# get five redis keys
self.keys = {k: 'aproxy:' + self.name + ':' + k for k in key_names}
self.pq_key = 'acrawler:' + self.name + ':' + 'q:pq'
self.score_limit = self.crawler.config.get('SCORE_LIMIT')
self.speed_limit = self.crawler.config.get('SPEED_LIMIT')
self.ttl_limit = self.crawler.config.get('TTL_LIMIT')
def __init__(self):
super().__init__(command_prefix=get_pre,
description=desc,
pm_help=None,
help_attrs=dict(hidden=True),
formatter=RubHelpFormatter(),
case_insensitive=True)
self.load_cogs()
self.add_command(self.source)
self.session = aiohttp.ClientSession(loop=self.loop)
self.db = db.DB(config.db_host, config.db_user, config.db_pass,
config.db_name, self.loop)
self.redis = self.loop.run_until_complete(
aioredis.create_redis_pool('redis://redis', loop=self.loop))
self.redis = aioredis.Redis(self.redis)
self.prefixes = {}
def __init__(self):
super().__init__(command_prefix=config.prefix.split(),
description=desc,
pm_help=None,
case_insensitive=True,
help_attrs=dict(hidden=True),
game=discord.Game(name='~help'),
formatter=KiaraFormatter())
self.load_cogs()
self.session = aiohttp.ClientSession(loop=self.loop)
self.db = db.DB(config.MYSQL_HOST, config.MYSQL_USER,
config.MYSQL_PASSWORD, config.MYSQL_DATABASE,
self.loop)
self.redis = self.loop.run_until_complete(
aioredis.create_redis_pool('redis://redis', loop=self.loop))
self.redis = aioredis.Redis(self.redis)
def __init__(self,
config: RedisConfig,
*,
loop: asyncio.BaseEventLoop = None):
"""
:param loop:
:type loop:
"""
BaseCache.__init__(self, config.expiration)
_loop = loop or asyncio.get_event_loop()
self._redis_pool = _loop.run_until_complete(
aioredis.create_redis_pool((config.host, config.port),
db=config.db,
password=config.password))
self.key_prefix = config.key_prefix
async def get_redis_client(poolsize=None):
redis_client = None
try:
host = TannerConfig.get('REDIS', 'host')
port = TannerConfig.get('REDIS', 'port')
if poolsize is None:
poolsize = TannerConfig.get('REDIS', 'poolsize')
timeout = TannerConfig.get('REDIS', 'timeout')
redis_client = await asyncio.wait_for(aioredis.create_redis_pool(
(host, int(port)), maxsize=int(poolsize)),
timeout=int(timeout))
except asyncio.TimeoutError as timeout_error:
LOGGER.exception(
'Problem with redis connection. Please, check your redis server. %s',
timeout_error)
exit()
return redis_client
def redis():
from testing.redis import RedisServer
server = RedisServer(
redis_conf={
'requirepass':
'******', # use password to make sure clients support using a password
'port': 0, # force using unix domain socket
'loglevel': 'warning' # suppress unnecessary messages
})
config = server.dsn()
connection = aioredis.create_redis_pool(config['unix_socket_path'],
password=config['password'],
db=config['db'],
encoding='utf-8')
set_global_connection(connection)
yield connection
connection.close()
server.stop()
def __init__(self, loop: asyncio.BaseEventLoop):
super().__init__(loop)
self.tasks = []
self.sockets_ingesting = {}
# Redis
self.redis = aioredis.create_redis_pool(
os.environ.get("REDIS_URL", "redis://localhost"),
minsize=5,
maxsize=69, # hehe 69
loop=loop,
)
# Setup HTTP session
conn = aiohttp.TCPConnector(limit=0) # Uncap the max HTTP connections.
self.session = aiohttp.ClientSession(
loop=loop,
headers={"User-Agent": "RPAN dumper sponsored by u/nepeat"},
connector=conn,
)
async def new(cls,
redis_address: str,
database_filename: str,
post_database_statement: str,
bind_address: str,
bind_port: int,
*,
debug: bool = False) -> Server:
redis_connection, conn = await asyncio.gather(
aioredis.create_redis_pool(redis_address),
UrlDatabase.new(database_filename,
post_database_statement,
renew=debug))
await redis_connection.delete('us_auth')
keys = [key async for key in conn.get_all_authorized_key()]
if keys:
await redis_connection.sadd('us_auth', *keys)
return cls(redis_connection,
conn,
bind_address,
bind_port,
debug=debug)
def __init__(self,
app: ASGIApp,
fapi: FastAPI,
secret_key: typing.Union[str, Secret],
session_cookie: str = "session",
max_age: int = 3000,
same_site: str = "lax",
https_only: bool = False,
dispatch: DispatchFunction = None) -> None:
super().__init__(app, dispatch)
self.redis = aioredis.create_redis_pool(settings.redis_url)
self.redis_inited = False
self.app = app
self.signer = itsdangerous.TimestampSigner(str(secret_key))
self.session_cookie = session_cookie
self.max_age = max_age
self.same_site = same_site
self.https_only = https_only
@fapi.on_event("shutdown")
async def shutdown():
self.redis.close()
await self.redis.wait_closed()
async def on_ready(self):
if self.prepared.is_set():
return
self.redis = await asyncio.wait_for(aioredis.create_redis_pool(
config.REDIS_ADDRESS, password=config.REDIS_PASS),
timeout=20.0)
LOG.info("Connected to Redis server.")
self.db = await asyncpg.create_pool(**config.ASYNCPG)
LOG.info("Connected to PostgreSQL server.")
for guild in self.guilds:
prefix = set(
map(bytes.decode, await
self.redis.smembers(f"prefixes_{guild.id}")))
if not prefix:
prefix = set(config.PREFIX)
self.prefixes[guild.id] = prefix
async with self.db.acquire() as cur:
with open("schema.sql") as file:
await cur.execute(file.read())
for userid, reason in await cur.fetch(
"SELECT user_id, reason FROM blacklist;"):
if not self.get_user(userid):
LOG.warning("Blacklisted ID \"%s\" is unknown.", userid)
# await cur.execute("DELETE FROM blacklist WHERE userid=$1;", userid)
self.blacklist[userid] = reason
LOG.info("User %s (%s) is blacklisted.", self.get_user(userid),
userid)
self.prepared.set()
LOG.info("Setup complete. Listening to commands on prefix \"%s\".",
config.PREFIX)
await self.change_presence(activity=discord.Game(
name="Use *tutorial to begin!"))
def get_async_redis(loop):
return loop.run_until_complete(
aioredis.create_redis_pool(redis_config.get('url'),
loop=loop,
encoding='utf-8'))
def pool(self):
return self.loop.run_until_complete(aioredis.create_redis_pool(
'redis://localhost'))
import asyncio
import aioredis
from structlog import get_logger
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
from tornado.options import define, options
from tornado.web import Application
from api.constants import REDIS_HOST, REDIS_PORT
from api.handlers import handlers
from init import create_consumer_group
if __name__ == '__main__':
define('port', default=3000, help='port to listen on')
application = Application(handlers)
application.log = get_logger()
server = HTTPServer(application)
server.bind(options.port)
server.start(0) # forks one process per cpu
loop = asyncio.get_event_loop()
application.redis = loop.run_until_complete(
aioredis.create_redis_pool((REDIS_HOST, REDIS_PORT), loop=loop))
asyncio.get_event_loop().run_until_complete(
create_consumer_group(application.redis))
loop.run_forever()
IOLoop.current().start()
def __init__(self, loop=None):
self.loop = loop if loop else asyncio.get_running_loop()
self.redis_client = self.loop.run_until_complete(
aioredis.create_redis_pool('redis://localhost'))
from phrases.phrase_handler import PhraseHandler
from phrases.phrase_types import PhraseTypes
from settings import setup_bot
from utils import bot_typing
DEBUG = int(os.getenv("DEBUG", "0")) == 1
bot, dispatcher = setup_bot(os.getenv("API_TOKEN"))
phrase_handler = PhraseHandler()
api_client = ApiClient(
api_url=f"http://{os.getenv('API_HOST')}:{os.getenv('API_PORT')}",
loop=bot.loop)
redis = bot.loop.run_until_complete(
aioredis.create_redis_pool(
f'redis://{os.getenv("REDIS_HOST")}:{os.getenv("REDIS_PORT")}'))
support_chat_id = int(os.getenv("SUPPORT_CHAT_ID"))
predictors = bot.loop.run_until_complete(api_client.get_all_predictors())
answer_cb = CallbackData('answer_callback', 'id', 'rating', 'predictor')
@dispatcher.message_handler(commands=["start"])
async def send_welcome(message: types.Message):
await bot_typing(bot, message.chat.id)
await bot.send_message(message.chat.id,
phrase_handler.get_phrase(
PhraseTypes.WELCOME_PHRASE),