def fetch_achievements_db():
"""Fetches all achievements from db on startup."""
# For fella who wants to use our new achievements system. You need database with content to fetch
# you can use cmyuis gulag one as our system was based on it.
# But for security reasons RealistikOsu will never share their database.
achievements = glob.db.fetchAll("SELECT * FROM new_achievements")
for achievement in achievements:
condition = eval(
f"lambda score, mode_vn, stats: {achievement.pop('cond')}")
glob.achievements.append(
Achievement(_id=achievement['id'],
file=achievement['file'],
name=achievement['name'],
desc=achievement['desc'],
cond=condition))
async def initialize_ram_caches(db_cursor: aiomysql.DictCursor) -> None:
"""Setup & cache the global collections before listening for connections."""
# dynamic (active) sets, only in ram
glob.matches = Matches()
glob.players = Players()
# static (inactive) sets, in ram & sql
glob.channels = await Channels.prepare(db_cursor)
glob.clans = await Clans.prepare(db_cursor)
glob.pools = await MapPools.prepare(db_cursor)
bot_name = await misc.utils.fetch_bot_name(db_cursor)
# create bot & add it to online players
glob.bot = Player(
id=1,
name=bot_name,
login_time=float(0x7FFFFFFF), # (never auto-dc)
priv=Privileges.NORMAL,
bot_client=True,
)
glob.players.append(glob.bot)
# global achievements (sorted by vn gamemodes)
glob.achievements = []
await db_cursor.execute("SELECT * FROM achievements")
async for row in db_cursor:
# NOTE: achievement conditions are stored as stringified python
# expressions in the database to allow for extensive customizability.
condition = eval(f'lambda score, mode_vn: {row.pop("cond")}')
achievement = Achievement(**row, cond=condition)
glob.achievements.append(achievement)
# static api keys
await db_cursor.execute(
"SELECT id, api_key FROM users WHERE api_key IS NOT NULL")
glob.api_keys = {row["api_key"]: row["id"] async for row in db_cursor}
async def setup_collections() -> None:
"""Setup & cache many global collections (mostly from sql)."""
glob.players = PlayerList() # online players
glob.matches = MatchList() # active multiplayer matches
glob.channels = await ChannelList.prepare() # active channels
glob.clans = await ClanList.prepare() # active clans
glob.pools = await MapPoolList.prepare() # active mappools
# create our bot & append it to the global player list.
res = await glob.db.fetch('SELECT name FROM users WHERE id = 1')
glob.bot = Player(
id = 1, name = res['name'], priv = Privileges.Normal,
login_time = float(0x7fffffff), # never auto-dc
bot_client = True
)
glob.players.append(glob.bot)
# global achievements (sorted by vn gamemodes)
glob.achievements = {0: [], 1: [], 2: [], 3: []}
async for row in glob.db.iterall('SELECT * FROM achievements'):
# NOTE: achievement conditions are stored as
# stringified python expressions in the database
# to allow for easy custom achievements.
condition = eval(f'lambda score: {row.pop("cond")}')
achievement = Achievement(**row, cond=condition)
# NOTE: achievements are grouped by modes internally.
glob.achievements[row['mode']].append(achievement)
# static api keys
glob.api_keys = {
row['api_key']: row['id']
for row in await glob.db.fetchall(
'SELECT id, api_key FROM users '
'WHERE api_key IS NOT NULL'
)
}
async def setup_collections() -> None:
"""Setup & cache many global collections."""
# dynamic (active) sets, only in ram
glob.players = Players()
glob.matches = Matches()
# static (inactive) sets, in ram & sql
glob.channels = await Channels.prepare()
glob.clans = await Clans.prepare()
glob.pools = await MapPools.prepare()
# create bot & add it to online players
glob.bot = Player(id=1,
name=await fetch_bot_name(),
priv=Privileges.Normal,
login_time=float(0x7fffffff),
bot_client=True) # never auto-dc the bot ^
glob.players.append(glob.bot)
# global achievements (sorted by vn gamemodes)
glob.achievements = {0: [], 1: [], 2: [], 3: []}
async for row in glob.db.iterall('SELECT * FROM achievements'):
# NOTE: achievement conditions are stored as
# stringified python expressions in the database
# to allow for easy custom achievements.
condition = eval(f'lambda score: {row.pop("cond")}')
achievement = Achievement(**row, cond=condition)
# NOTE: achievements are grouped by modes internally.
glob.achievements[row['mode']].append(achievement)
# static api keys
glob.api_keys = {
row['api_key']: row['id']
for row in await glob.db.fetchall('SELECT id, api_key FROM users '
'WHERE api_key IS NOT NULL')
}
async def connect(
) -> None: # ran before server startup, used to do things like connecting to mysql :D
info(f"Asahi v{glob.version} starting")
glob.web = ClientSession() # aiohttp session for external web requests
from lists.players import PlayerList
glob.players = PlayerList() # init player list
try:
glob.db = await fatFawkSQL.connect(**glob.config.sql
) # connect to db using config :p
debug("Asahi connected to MySQL")
except Exception:
error(f"Asahi failed to connect to MySQL\n\n{traceback.format_exc()}")
raise SystemExit(1)
try:
glob.redis = await aioredis.create_redis_pool(
f"redis://{glob.config.redis['host']}",
db=glob.config.redis["db"],
password=glob.config.redis["password"] or None,
)
debug("Asahi connected to Redis")
except Exception:
error(f"Asahi failed to connect to Redis\n\n{traceback.format_exc()}")
raise SystemExit(1)
from objects.player import Player
botinfo = await glob.db.fetchrow(
"SELECT name, pw, country, name FROM users WHERE id = 1", )
if not botinfo:
error(
"Bot account not found. "
"Please insert the bot account with user ID 1 and start Asahi again"
)
raise SystemExit(1)
glob.bot = Player(
id=1,
name=botinfo["name"],
offset=1,
country_iso=botinfo["country"],
country=country_codes[botinfo["country"].upper()],
)
await glob.bot.set_stats()
glob.players.append(glob.bot)
debug(f"Added bot {glob.bot.name} to player list")
async for ach_row in glob.db.iter("SELECT * FROM achievements"):
ach_row["cond"] = eval(f'lambda s: {ach_row["cond"]}')
ach_row["desc"] = ach_row.pop("descr")
glob.achievements.append(Achievement(**ach_row))
init_customs() # set custom achievements list for assets proxy
# add all channels to cache
from objects.channel import Channel
async for chan_row in glob.db.iter("SELECT * FROM channels"):
chan_row["desc"] = chan_row.pop("descr")
channel = Channel(**chan_row)
glob.channels[channel.name] = channel
debug(f"Added channel {channel.name} to channel list")
# add announce channel to cache
announce = Channel(
name="#announce",
desc="#1 scores and public announcements will be posted here",
auto=True,
perm=True,
)
glob.channels[announce.name] = announce
debug("Added channel #announce to channel list")
# add lobby channel to cache
lobby = Channel(name="#lobby",
desc="Multiplayer lobby",
auto=False,
perm=True)
glob.channels[lobby.name] = lobby
debug("Added channel #lobby to channel list")
# add all clans to cache
async for clan_row in glob.db.iter("SELECT * FROM clans"):
clan = Clan(**clan_row)
clan_chan = Channel(
name="#clan",
desc=f"Clan chat for clan {clan.name}",
auto=False,
perm=True,
)
clan.chan = clan_chan # uwu
glob.clans[clan.id] = clan
clan.country = await glob.db.fetchval(
"SELECT country FROM users WHERE id = %s",
[clan.owner],
)
async for member_row in glob.db.iter(
"SELECT id FROM users WHERE clan = %s",
[clan.id],
):
clan.members.append(member_row["id"])
await glob.redis.zadd(f"asahi:clan_leaderboard", clan.score, clan.id)
await glob.redis.zadd(
f"asahi:clan_leaderboard:{clan.country}",
clan.score,
clan.id,
)
r = await glob.redis.zrevrank(f"asahi:clan_leaderboard", clan.id)
cr = await glob.redis.zrevrank(
f"asahi:clan_leaderboard:{clan.country}",
clan.id,
)
clan.rank = r + 1 if r else 0
clan.country_rank = cr + 1 if cr else 0
debug(f"Added clan {clan.name} to clan list")
await prepare_tasks() # make new db conn for donor/freeze tasks
glob.app.add_task(expired_donor)
glob.app.add_task(freeze_timers)
info(f"Asahi v{glob.version} started")
def main() -> int:
# AGPL license agreement
try:
agpl.check_license("ripple", "LETS")
except agpl.LicenseError as e:
print(str(e))
return 1
try:
consoleHelper.printServerStartHeader(True)
# Read config
consoleHelper.printNoNl("> Reading config file... ")
glob.conf = config.config("config.ini")
if glob.conf.default:
# We have generated a default config.ini, quit server
consoleHelper.printWarning()
consoleHelper.printColored(
"[!] config.ini not found. A default one has been generated.",
bcolors.YELLOW)
consoleHelper.printColored(
"[!] Please edit your config.ini and run the server again.",
bcolors.YELLOW)
return 1
# If we haven't generated a default config.ini, check if it's valid
if not glob.conf.checkConfig():
consoleHelper.printError()
consoleHelper.printColored(
"[!] Invalid config.ini. Please configure it properly",
bcolors.RED)
consoleHelper.printColored(
"[!] Delete your config.ini to generate a default one",
bcolors.RED)
return 1
else:
consoleHelper.printDone()
# Read additional config file
consoleHelper.printNoNl("> Loading additional config file... ")
try:
if not os.path.isfile(glob.conf.config["custom"]["config"]):
consoleHelper.printWarning()
consoleHelper.printColored(
"[!] Missing config file at {}; A default one has been generated at this location."
.format(glob.conf.config["custom"]["config"]),
bcolors.YELLOW)
shutil.copy("common/default_config.json",
glob.conf.config["custom"]["config"])
with open(glob.conf.config["custom"]["config"], "r") as f:
glob.conf.extra = json.load(f)
consoleHelper.printDone()
except:
consoleHelper.printWarning()
consoleHelper.printColored(
"[!] Unable to load custom config at {}".format(
glob.conf.config["custom"]["config"]), bcolors.RED)
return 1
# Create data/oppai maps folder if needed
consoleHelper.printNoNl("> Checking folders... ")
paths = [
".data", ".data/oppai", ".data/catch_the_pp",
glob.conf.config["server"]["replayspath"],
"{}_relax".format(glob.conf.config["server"]["replayspath"]),
glob.conf.config["server"]["beatmapspath"],
glob.conf.config["server"]["screenshotspath"]
]
for i in paths:
if not os.path.exists(i):
os.makedirs(i, 0o770)
consoleHelper.printDone()
# Connect to db
try:
consoleHelper.printNoNl("> Connecting to MySQL database... ")
glob.db = dbConnector.db(glob.conf.config["db"]["host"],
glob.conf.config["db"]["username"],
glob.conf.config["db"]["password"],
glob.conf.config["db"]["database"],
int(glob.conf.config["db"]["workers"]))
consoleHelper.printNoNl(" ")
consoleHelper.printDone()
except:
# Exception while connecting to db
consoleHelper.printError()
consoleHelper.printColored(
"[!] Error while connection to database. Please check your config.ini and run the server again",
bcolors.RED)
raise
# Connect to redis
try:
consoleHelper.printNoNl("> Connecting to redis... ")
glob.redis = redis.Redis(glob.conf.config["redis"]["host"],
glob.conf.config["redis"]["port"],
glob.conf.config["redis"]["database"],
glob.conf.config["redis"]["password"])
glob.redis.ping()
consoleHelper.printNoNl(" ")
consoleHelper.printDone()
except:
# Exception while connecting to db
consoleHelper.printError()
consoleHelper.printColored(
"[!] Error while connection to redis. Please check your config.ini and run the server again",
bcolors.RED)
raise
# Empty redis cache
#TODO: do we need this?
try:
glob.redis.eval(
"return redis.call('del', unpack(redis.call('keys', ARGV[1])))",
0, "lets:*")
except redis.exceptions.ResponseError:
# Script returns error if there are no keys starting with peppy:*
pass
# Save lets version in redis
glob.redis.set("lets:version", glob.VERSION)
# Create threads pool
try:
consoleHelper.printNoNl("> Creating threads pool... ")
glob.pool = ThreadPool(int(glob.conf.config["server"]["threads"]))
consoleHelper.printDone()
except:
consoleHelper.printError()
consoleHelper.printColored(
"[!] Error while creating threads pool. Please check your config.ini and run the server again",
bcolors.RED)
# Load achievements
consoleHelper.printNoNl("> Loading achievements... ")
try:
achievements = glob.db.fetchAll("SELECT * FROM achievements")
for achievement in achievements:
condition = eval(
f"lambda score, mode_vn, stats: {achievement.pop('cond')}")
glob.achievements.append(
Achievement(_id=achievement['id'],
file=achievement['icon'],
name=achievement['name'],
desc=achievement['description'],
cond=condition))
except Exception as e:
consoleHelper.printError()
consoleHelper.printColored(
"[!] Error while loading achievements! ({})".format(
traceback.format_exc()),
bcolors.RED,
)
return 1
consoleHelper.printDone()
# Set achievements version
glob.redis.set("lets:achievements_version", glob.ACHIEVEMENTS_VERSION)
consoleHelper.printColored(
"Achievements version is {}".format(glob.ACHIEVEMENTS_VERSION),
bcolors.YELLOW)
# Print disallowed mods into console (Used to also assign it into variable but has been moved elsewhere)
unranked_mods = [
key for key, value in glob.conf.extra["common"]
["rankable-mods"].items() if not value
]
consoleHelper.printColored(
"Unranked mods: {}".format(", ".join(unranked_mods)),
bcolors.YELLOW)
# Print allowed beatmap rank statuses
allowed_beatmap_rank = [
key for key, value in glob.conf.extra["lets"]
["allowed-beatmap-rankstatus"].items() if value
]
consoleHelper.printColored(
"Allowed beatmap rank statuses: {}".format(
", ".join(allowed_beatmap_rank)), bcolors.YELLOW)
# Make array of bools to respective rank id's
glob.conf.extra["_allowed_beatmap_rank"] = [
getattr(rankedStatuses, key) for key in allowed_beatmap_rank
] # Store the allowed beatmap rank id's into glob
# Discord
if generalUtils.stringToBool(glob.conf.config["discord"]["enable"]):
glob.schiavo = schiavo.schiavo(
glob.conf.config["discord"]["boturl"], "**lets**")
else:
consoleHelper.printColored(
"[!] Warning! Discord logging is disabled!", bcolors.YELLOW)
# Check debug mods
glob.debug = generalUtils.stringToBool(
glob.conf.config["server"]["debug"])
if glob.debug:
consoleHelper.printColored(
"[!] Warning! Server running in debug mode!", bcolors.YELLOW)
# Server port
try:
serverPort = int(glob.conf.config["server"]["port"])
except:
consoleHelper.printColored(
"[!] Invalid server port! Please check your config.ini and run the server again",
bcolors.RED)
# Make app
glob.application = make_app()
# Set up sentry
try:
glob.sentry = generalUtils.stringToBool(
glob.conf.config["sentry"]["enable"])
if glob.sentry:
glob.application.sentry_client = AsyncSentryClient(
glob.conf.config["sentry"]["dsn"], release=glob.VERSION)
else:
consoleHelper.printColored(
"[!] Warning! Sentry logging is disabled!", bcolors.YELLOW)
except:
consoleHelper.printColored(
"[!] Error while starting Sentry client! Please check your config.ini and run the server again",
bcolors.RED)
# Set up Datadog
try:
if generalUtils.stringToBool(
glob.conf.config["datadog"]["enable"]):
glob.dog = datadogClient.datadogClient(
glob.conf.config["datadog"]["apikey"],
glob.conf.config["datadog"]["appkey"])
else:
consoleHelper.printColored(
"[!] Warning! Datadog stats tracking is disabled!",
bcolors.YELLOW)
except:
consoleHelper.printColored(
"[!] Error while starting Datadog client! Please check your config.ini and run the server again",
bcolors.RED)
# Connect to pubsub channels
pubSub.listener(glob.redis, {
"lets:beatmap_updates": beatmapUpdateHandler.handler(),
}).start()
# Prometheus port
statsPort = None
try:
if glob.conf.config["prometheus"]["port"]:
statsPort = int(glob.conf.config["prometheus"]["port"])
except:
consoleHelper.printColored(
"Invalid stats port! Please check your config.ini and run the server again",
bcolors.YELLOW)
raise
if statsPort:
consoleHelper.printColored(
"Stats exporter listening on localhost:{}".format(statsPort),
bcolors.GREEN)
prometheus_client.start_http_server(statsPort, addr="127.0.0.1")
# Server start message and console output
consoleHelper.printColored(
"> L.E.T.S. is listening for clients on {}:{}...".format(
glob.conf.config["server"]["host"], serverPort), bcolors.GREEN)
# Start Tornado
glob.application.listen(serverPort,
address=glob.conf.config["server"]["host"])
tornado.ioloop.IOLoop.instance().start()
finally:
# Perform some clean up
print("> Disposing server... ")
glob.fileBuffers.flushAll()
consoleHelper.printColored("Goodbye!", bcolors.GREEN)
return 0
async def setup_collections() -> None:
"""Setup & cache many global collections (mostly from sql)."""
# create our bot & append it to the global player list.
res = await glob.db.fetch('SELECT name FROM users WHERE id = 1')
# global players list
glob.players = PlayerList()
glob.bot = Player(
id=1,
name=res['name'],
priv=Privileges.Normal,
last_recv_time=float(0x7fffffff) # never auto-dc
)
glob.players.append(glob.bot)
# global channels list
glob.channels = ChannelList()
async for res in glob.db.iterall('SELECT * FROM channels'):
chan = Channel(name=res['name'],
topic=res['topic'],
read_priv=Privileges(res['read_priv']),
write_priv=Privileges(res['write_priv']),
auto_join=res['auto_join'] == 1)
glob.channels.append(chan)
# global matches list
glob.matches = MatchList()
# global clans list
glob.clans = ClanList()
async for res in glob.db.iterall('SELECT * FROM clans'):
clan = Clan(**res)
await clan.members_from_sql()
glob.clans.append(clan)
# global mappools list
glob.pools = MapPoolList()
async for res in glob.db.iterall('SELECT * FROM tourney_pools'):
pool = MapPool(id=res['id'],
name=res['name'],
created_at=res['created_at'],
created_by=await
glob.players.get_ensure(id=res['created_by']))
await pool.maps_from_sql()
glob.pools.append(pool)
# global achievements (sorted by vn gamemodes)
glob.achievements = {0: [], 1: [], 2: [], 3: []}
async for res in glob.db.iterall(
'SELECT `id`, `file`, `name`, `desc`, `cond`, `mode` FROM achievements'
):
# NOTE: achievement conditions are stored as
# stringified python expressions in the database
# to allow for easy custom achievements.
condition = eval(f'lambda score: {res.pop("cond")}')
achievement = Achievement(**res, cond=condition)
# NOTE: achievements are grouped by modes internally.
glob.achievements[res['mode']].append(achievement)
""" XXX: Unfinished code for beatmap submission.
async def on_start() -> None:
glob.http = aiohttp.ClientSession(json_serialize=orjson.dumps)
# connect to mysql
glob.db = cmyui.AsyncSQLPool()
await glob.db.connect(glob.config.mysql)
# run the sql updater
updater = Updater(glob.version)
await updater.run()
await updater.log_startup()
# create our bot & append it to the global player list.
glob.bot = Player(id=1, name='Aika', priv=Privileges.Normal)
glob.bot.last_recv_time = float(0x7fffffff)
glob.players.append(glob.bot)
# TODO: this section is getting a bit gross.. :P
# should be moved and probably refactored pretty hard.
# add all channels from db.
async for c_res in glob.db.iterall('SELECT * FROM channels'):
c_res['read_priv'] = Privileges(c_res.get('read_priv', 1))
c_res['write_priv'] = Privileges(c_res.get('write_priv', 2))
glob.channels.append(Channel(**c_res))
# add all mappools from db.
async for p_res in glob.db.iterall('SELECT * FROM tourney_pools'):
# overwrite basic types with some class types
creator = await glob.players.get(id=p_res['created_by'], sql=True)
p_res['created_by'] = creator # replace id with player object
pool = MapPool(**p_res)
await pool.maps_from_sql()
glob.pools.append(pool)
# add all clans from db.
async for c_res in glob.db.iterall('SELECT * FROM clans'):
# fetch all members from sql
m_res = await glob.db.fetchall(
'SELECT id, clan_rank '
'FROM users '
'WHERE clan_id = %s',
c_res['id'],
_dict=False)
members = set()
for p_id, clan_rank in m_res:
if clan_rank == 3:
c_res['owner'] = p_id
members.add(p_id)
glob.clans.append(Clan(**c_res, members=members))
# add all achievements from db.
async for a_res in glob.db.iterall('SELECT * FROM achievements'):
condition = eval(f'lambda score: {a_res.pop("cond")}')
achievement = Achievement(**a_res, cond=condition)
glob.achievements[a_res['mode']].append(achievement)
""" bmsubmit stuff
# get the latest set & map id offsets for custom maps.
maps_res = await glob.db.fetch(
'SELECT id, set_id FROM maps '
'WHERE server = "gulag" '
'ORDER BY id ASC LIMIT 1'
)
if maps_res:
glob.gulag_maps = maps_res
"""
# add new donation ranks & enqueue tasks to remove current ones.
# TODO: this system can get quite a bit better; rather than just
# removing, it should rather update with the new perks (potentially
# a different tier, enqueued after their current perks).
async def rm_donor(userid: int, delay: int):
await asyncio.sleep(delay)
p = await glob.players.get(id=userid, sql=True)
await p.remove_privs(Privileges.Donator)
log(f"{p}'s donation perks have expired.", Ansi.MAGENTA)
query = ('SELECT id, donor_end FROM users '
'WHERE donor_end > UNIX_TIMESTAMP()')
async for donation in glob.db.iterall(query):
# calculate the delta between now & the exp date.
delta = donation['donor_end'] - time.time()
if delta > (60 * 60 * 24 * 30):
# ignore donations expiring in over a months time;
# the server should restart relatively often anyways.
continue
asyncio.create_task(rm_donor(donation['id'], delta))