async def start(self):
runner = AppRunner(self.buildApp(Application()))
await runner.setup()
site = TCPSite(runner, "0.0.0.0", int(self.config.get(Setting.PORT)))
await site.start()
self.logger.info("Backup Auth Server Started")
async def obtain_access_token(studip_session: StudIPSession,
oauth1_client: OAuth1Client = None,
http_session: aiohttp.ClientSession = None,
port=OAUTH_CALLBACK_PORT,
open_browser=True):
if not http_session:
http_session = studip_session.http.http_session
if not oauth1_client:
if isinstance(http_session._default_auth, OAuth1Client):
oauth1_client = http_session._default_auth
else:
oauth1_client = OAuth1Client(
*get_tokens(studip_session.studip_base))
http_session._default_auth = oauth1_client
try:
await studip_session.check_login()
return oauth1_client
except (HTTPException, aiohttp.ClientResponseError) as e:
if e.status != HTTPUnauthorized.status_code \
and getattr(e, "message", "") != "Can't verify request, missing oauth_consumer_key or oauth_token":
raise
log.info("OAuth Session invalid, starting log-in flow")
app = Application()
app.add_routes(routes)
app["http_session"] = http_session
app["studip_session"] = studip_session
app["oauth1_client"] = oauth1_client
app["finished_client_future"] = asyncio.get_event_loop().create_future()
app["check_running_future"] = asyncio.get_event_loop().create_future()
runner = AppRunner(app)
try:
await runner.setup()
site = TCPSite(runner, host="127.0.0.1", port=port)
await site.start()
nonce = generate_nonce()
log.debug("nonce %s", nonce)
async with http_session.get(
url_for("check_running", app=app,
site=site).with_query(nonce=nonce)) as resp:
resp.raise_for_status()
assert await resp.text() == "running"
assert await asyncio.wait_for(app["check_running_future"],
0) == nonce
log.debug("server check passed")
start_addr = str(url_for("start_oauth", app=app, site=site))
log.info("Go to the following address to log in via OAuth1: %s",
start_addr)
if open_browser is True:
import webbrowser
open_browser = webbrowser.open
if callable(open_browser):
await asyncio.get_event_loop().run_in_executor(
None, lambda: open_browser(start_addr)),
return await app["finished_client_future"]
finally:
await runner.cleanup()
def run_website(args: argparse.Namespace) -> None:
"""
Starts the website, connects the database, logs in the specified bots, runs the async loop forever.
Parameters
-----------
args: :class:`argparse.Namespace`
The arguments namespace that wants to be run.
"""
# Load our imports here so we don't need to require them all the time
from aiohttp.web import Application, AppRunner, TCPSite
from aiohttp_jinja2 import setup as jinja_setup
from aiohttp_session import setup as session_setup, SimpleCookieStorage
from aiohttp_session.cookie_storage import EncryptedCookieStorage as ECS
from jinja2 import FileSystemLoader
import re
import html
from datetime import datetime as dt
import markdown
os.chdir(args.website_directory)
set_event_loop()
# Read config
with open(args.config_file) as a:
config = toml.load(a)
# Create website object - don't start based on argv
app = Application(loop=asyncio.get_event_loop(), debug=args.debug)
app['static_root_url'] = '/static'
for route in config['routes']:
module = importlib.import_module(f"website.{route}", "temp")
app.router.add_routes(module.routes)
app.router.add_static('/static', os.getcwd() + '/website/static', append_version=True)
# Add middlewares
if args.debug:
session_setup(app, SimpleCookieStorage(max_age=1_000_000))
else:
session_setup(app, ECS(os.urandom(32), max_age=1_000_000))
jinja_env = jinja_setup(app, loader=FileSystemLoader(os.getcwd() + '/website/templates'))
# Add our jinja env filters
def regex_replace(string, find, replace):
return re.sub(find, replace, string, re.IGNORECASE | re.MULTILINE)
def escape_text(string):
return html.escape(string)
def timestamp(string):
return dt.fromtimestamp(float(string))
def int_to_hex(string):
return format(hex(int(string))[2:], "0>6")
def to_markdown(string):
return markdown.markdown(string, extensions=['extra'])
def display_mentions(string, users):
def get_display_name(group):
user = users.get(group.group('userid'))
if not user:
return 'unknown-user'
return user.get('display_name') or user.get('username')
return re.sub(
'(?:<|(?:<))@!?(?P<userid>\\d{16,23})(?:>|(?:>))',
lambda g: f'<span class="chatlog__mention">@{get_display_name(g)}</span>',
string,
re.IGNORECASE | re.MULTILINE,
)
def display_emojis(string):
def get_html(group):
return (
f'<img class="discord_emoji" src="https://cdn.discordapp.com/emojis/{group.group("id")}'
f'.{"gif" if group.group("animated") else "png"}" alt="Discord custom emoji: '
f'{group.group("name")}" style="height: 1em; width: auto;">'
)
return re.sub(
r"(?P<emoji>(?:<|<)(?P<animated>a)?:(?P<name>\w+):(?P<id>\d+)(?:>|>))",
get_html,
string,
re.IGNORECASE | re.MULTILINE,
)
jinja_env.filters['regex_replace'] = regex_replace
jinja_env.filters['escape_text'] = escape_text
jinja_env.filters['timestamp'] = timestamp
jinja_env.filters['int_to_hex'] = int_to_hex
jinja_env.filters['markdown'] = to_markdown
jinja_env.filters['display_mentions'] = display_mentions
jinja_env.filters['display_emojis'] = display_emojis
# Add our connections and their loggers
app['database'] = DatabaseWrapper
app['redis'] = RedisConnection
app['logger'] = logger.getChild("route")
app['stats'] = StatsdConnection
# Add our config
app['config'] = config
loop = app.loop
# Connect the database pool
if app['config'].get('database', {}).get('enabled', False):
db_connect_task = start_database_pool(app['config'])
loop.run_until_complete(db_connect_task)
# Connect the redis pool
if app['config'].get('redis', {}).get('enabled', False):
re_connect = start_redis_pool(app['config'])
loop.run_until_complete(re_connect)
# Add our bots
app['bots'] = {}
for index, (bot_name, bot_config_location) in enumerate(config.get('discord_bot_configs', dict()).items()):
bot = Bot(f"./config/{bot_config_location}")
app['bots'][bot_name] = bot
if index == 0:
set_default_log_levels(args)
try:
loop.run_until_complete(bot.login())
bot.load_all_extensions()
except Exception:
logger.error(f"Failed to start bot {bot_name}", exc_info=True)
exit(1)
# Start the HTTP server
logger.info("Creating webserver...")
application = AppRunner(app)
loop.run_until_complete(application.setup())
webserver = TCPSite(application, host=args.host, port=args.port)
# Start the webserver
loop.run_until_complete(webserver.start())
logger.info(f"Server started - http://{args.host}:{args.port}/")
# This is the forever loop
try:
logger.info("Running webserver")
loop.run_forever()
except KeyboardInterrupt:
pass
# We're now done running the bot, time to clean up and close
loop.run_until_complete(application.cleanup())
if config.get('database', {}).get('enabled', False):
logger.info("Closing database pool")
try:
if DatabaseWrapper.pool:
loop.run_until_complete(asyncio.wait_for(DatabaseWrapper.pool.close(), timeout=30.0))
except asyncio.TimeoutError:
logger.error("Couldn't gracefully close the database connection pool within 30 seconds")
if config.get('redis', {}).get('enabled', False):
logger.info("Closing redis pool")
RedisConnection.pool.close()
logger.info("Closing asyncio loop")
loop.stop()
loop.close()
def _new_transport_tcp(self) -> BaseSite:
return TCPSite(
self._runner,
self.app.conf.web_bind,
self.app.conf.web_port,
)
async def run_web_app(cls,app,host,port):
runner = AppRunner(app)
await runner.setup()
site = TCPSite(runner, host,port)
await site.start()
def run_interactions(args: argparse.Namespace) -> None:
"""
Starts the bot, connects the database, runs the async loop forever.
Parameters
-----------
args: :class:`argparse.Namespace`
The arguments namespace that wants to be run.
"""
from aiohttp.web import Application, AppRunner, TCPSite
os.chdir(args.bot_directory)
set_event_loop()
# And run file
bot = Bot(config_file=args.config_file, intents=discord.Intents.none())
loop = bot.loop
EventLoopCallbackHandler.bot = bot
# Set up loggers
bot.logger = logger.getChild("bot")
set_default_log_levels(args)
# Connect the database pool
if bot.config.get('database', {}).get('enabled', False):
db_connect_task = start_database_pool(bot.config)
loop.run_until_complete(db_connect_task)
# Connect the redis pool
if bot.config.get('redis', {}).get('enabled', False):
re_connect = start_redis_pool(bot.config)
loop.run_until_complete(re_connect)
# Load the bot's extensions
logger.info('Loading extensions... ')
bot.load_all_extensions()
# Run the bot
logger.info("Logging in bot")
loop.run_until_complete(bot.login())
websocket_task = None
if args.connect:
logger.info("Connecting bot to gateway")
websocket_task = loop.create_task(bot.connect())
# Create the webserver
app = Application(loop=asyncio.get_event_loop(), debug=args.debug)
app.router.add_routes(commands.get_interaction_route_table(bot, bot.config.get("pubkey", ""), path=args.path))
# Start the HTTP server
logger.info("Creating webserver...")
application = AppRunner(app)
loop.run_until_complete(application.setup())
webserver = TCPSite(application, host=args.host, port=args.port)
# Start the webserver
loop.run_until_complete(webserver.start())
logger.info(f"Server started - http://{args.host}:{args.port}/")
# This is the forever loop
try:
logger.info("Running webserver")
loop.run_forever()
except KeyboardInterrupt:
pass
# We're now done running the webserver, time to clean up and close
if websocket_task:
websocket_task.cancel()
if bot.config.get('database', {}).get('enabled', False):
logger.info("Closing database pool")
try:
if DatabaseWrapper.pool:
loop.run_until_complete(asyncio.wait_for(DatabaseWrapper.pool.close(), timeout=30.0))
except asyncio.TimeoutError:
logger.error("Couldn't gracefully close the database connection pool within 30 seconds")
if bot.config.get('redis', {}).get('enabled', False):
logger.info("Closing redis pool")
RedisConnection.pool.close()
logger.info("Closing asyncio loop")
loop.stop()
loop.close()
utils.DatabaseConnection.create_pool(app['config']['database']))
# Connect the redis pool
if app['config'].get('redis', {}).get('enabled', True):
logger.info("Creating redis pool")
loop.run_until_complete(
utils.RedisConnection.create_pool(app['config']['redis']))
# Start the server unless I said otherwise
webserver = None
# HTTP server
logger.info("Creating webserver...")
application = AppRunner(app)
loop.run_until_complete(application.setup())
webserver = TCPSite(application, host=args.host, port=args.port)
# Start server
loop.run_until_complete(webserver.start())
logger.info(f"Server started - http://{args.host}:{args.port}/")
# This is the forever loop
try:
logger.info("Running webserver")
loop.run_forever()
except KeyboardInterrupt:
pass
# Clean up our shit
logger.info("Closing webserver")
loop.run_until_complete(application.cleanup())
loop = app.loop
logger.info("Creating database pool")
loop.run_until_complete(
DatabaseConnection.create_pool(app['config']['database']))
# Start the server unless I said otherwise
webserver = None
ssl_webserver = None
# HTTP server
logger.info("Creating webserver...")
application = AppRunner(app)
loop.run_until_complete(application.setup())
webserver = TCPSite(application, host=args.host, port=args.port)
# SSL server
try:
if not args.nossl:
ssl_context = SSLContext()
ssl_context.load_cert_chain(**app['config']['ssl_context'])
ssl_webserver = TCPSite(application,
host=args.host,
port=args.sslport,
ssl_context=ssl_context)
except Exception as e:
ssl_webserver = None
logger.exception("Could not make SSL webserver")
# Start servers
async def run_app(host: str, port: int) -> None:
runner = AppRunner(app)
await runner.setup()
site = TCPSite(runner, host, port)
await site.start()
async def test_cuncurrent_transactions(event_loop, unused_tcp_port):
"""
This test tests aiohttp_json_rpc.django.patch_db_connections.
The test sets up a JsonRpc and starts two concurrent clients.
Both clients try to open a Django transaction.
The test is successful if only the first client is able to commit its
database changes.
Note:
The transactions are slowed down artificial and Client #2 sleeps 200ms
on start using asyncio.sleep to make the test log more readable and
ensure the transactions are concurrent.
Note:
The test starts an watchdog to make sure the test does not hang if the
rpc communication is broken or hanging.
"""
from django_project.models import Item
from aiohttp_json_rpc.django import patch_db_connections
def create_client(client_id, *args, **kwargs):
url = 'http://localhost:{}'.format(unused_tcp_port)
future = asyncio.ensure_future(client(client_id, url, *args, **kwargs))
future.client_id = client_id
return future
patch_db_connections()
# just to be sure
assert Item.objects.count() == 0
# setup rpc
app = Application()
rpc = JsonRpc()
rpc.add_methods(
('', add),
)
app.router.add_route('*', '/', rpc.handle_request)
runner = AppRunner(app)
await runner.setup()
site = TCPSite(runner, 'localhost', unused_tcp_port)
await site.start()
# setup clients and watchdog
tasks = [
create_client(1, list(range(0, 10))),
create_client(2, list(range(2, 5)), sleep=0.2),
]
tasks = [
*tasks,
asyncio.ensure_future(watchdog(tasks)),
]
# run
await asyncio.gather(*tasks)
# checks
assert Item.objects.filter(client_id=1).count() == 10
assert not Item.objects.filter(client_id=2).exists()
assert tasks[0].result() == 0 # client #1
assert tasks[1].result() == 1 # client #2
assert tasks[2].result() == 0 # watchdog
if __name__ == '__main__':
'''
Starts the bot (and webserver if specified) and runs forever
'''
loop = bot.loop
print("Starting bot...")
bot.loop.create_task(bot.start_all())
if not args.noserver:
print("Starting server...")
web_runner = AppRunner(app)
loop.run_until_complete(web_runner.setup())
site = TCPSite(web_runner, args.host, args.port)
loop.run_until_complete(site.start())
print(f"Server started: http://{args.host}:{args.port}/")
# Store stuff in the bot for later
bot.web_runner = web_runner
# This is the forever loop
try:
loop.run_forever()
except (Exception, KeyboardInterrupt):
pass
finally:
# Logout the bot
loop.run_until_complete(bot.logout())