async def fetch_geoloc_web(ip: IPAddress) -> Optional[Geolocation]:
"""Fetch geolocation data based on ip (using ip-api)."""
url = f"http://ip-api.com/line/{ip}"
async with http.get(url) as resp:
if not resp or resp.status != 200:
log("Failed to get geoloc data: request failed.", Ansi.LRED)
return
status, *lines = (await resp.text()).split("\n")
if status != "success":
err_msg = lines[0]
if err_msg == "invalid query":
err_msg += f" ({url})"
log(f"Failed to get geoloc data: {err_msg}.", Ansi.LRED)
return
acronym = lines[1].lower()
return {
"latitude": float(lines[6]),
"longitude": float(lines[7]),
"country": {
"acronym": acronym,
"numeric": country_codes[acronym],
},
}
async def donor_expiry() -> list[Coroutine]:
"""Add new donation ranks & enqueue tasks to remove current ones."""
# TODO: this system can get quite a bit better; rather than just
# removing, it should rather update with the new perks (potentially
# a different tier, enqueued after their current perks).
async def rm_donor(userid: int, when: int):
if (delta := when - time.time()) >= 0:
await asyncio.sleep(delta)
p = await glob.players.get_ensure(id=userid)
# TODO: perhaps make a `revoke_donor` method?
await p.remove_privs(Privileges.Donator)
await glob.db.execute(
'UPDATE users '
'SET donor_end = 0 '
'WHERE id = %s',
[p.id]
)
if p.online:
p.enqueue(packets.notification('Your supporter status has expired.'))
log(f"{p}'s supporter status has expired.", Ansi.LMAGENTA)
def ensure_local_services_are_running() -> int:
"""Ensure all required services (mysql, redis) are running."""
# NOTE: if you have any problems with this, please contact me
# @cmyui#0425/[email protected]. i'm interested in knowing
# how people are using the software so that i can keep it
# in mind while developing new features & refactoring.
if settings.DB_DSN.hostname in ("localhost", "127.0.0.1", None):
# sql server running locally, make sure it's running
for service in ("mysqld", "mariadb"):
if os.path.exists(f"/var/run/{service}/{service}.pid"):
break
else:
# not found, try pgrep
pgrep_exit_code = subprocess.call(
["pgrep", "mysqld"],
stdout=subprocess.DEVNULL,
)
if pgrep_exit_code != 0:
log("Unable to connect to mysql server.", Ansi.LRED)
return 1
if not os.path.exists("/var/run/redis/redis-server.pid"):
log("Unable to connect to redis server.", Ansi.LRED)
return 1
return 0
def _download_achievement_images_osu(achievements_path: Path) -> bool:
"""Download all used achievement images (one by one, from osu!)."""
achs: list[str] = []
for res in ("", "@2x"):
for gm in ("osu", "taiko", "fruits", "mania"):
# only osu!std has 9 & 10 star pass/fc medals.
for n in range(1, 1 + (10 if gm == "osu" else 8)):
achs.append(f"{gm}-skill-pass-{n}{res}.png")
achs.append(f"{gm}-skill-fc-{n}{res}.png")
for n in (500, 750, 1000, 2000):
achs.append(f"osu-combo-{n}{res}.png")
log("Downloading achievement images from osu!.", Ansi.LCYAN)
for ach in achs:
resp = requests.get(f"https://assets.ppy.sh/medals/client/{ach}")
if resp.status_code != 200:
return False
log(f"Saving achievement: {ach}", Ansi.LCYAN)
(achievements_path / ach).write_bytes(resp.content)
return True
def _excepthook(
type_: Type[BaseException],
value: BaseException,
traceback: types.TracebackType,
):
if type_ is KeyboardInterrupt:
print("\33[2K\r", end="Aborted startup.")
return
elif type_ is AttributeError and value.args[0].startswith(
"module 'config' has no attribute", ):
attr_name = value.args[0][34:-1]
log(
"gulag's config has been updated, and has "
f"added a new `{attr_name}` attribute.",
Ansi.LMAGENTA,
)
log(
"Please refer to it's value & example in "
"ext/config.sample.py for additional info.",
Ansi.LCYAN,
)
return
printc(
f"gulag v{settings.VERSION} ran into an issue before starting up :(",
Ansi.RED,
)
real_excepthook(type_, value, traceback) # type: ignore
async def on_startup() -> None:
app.state.loop = asyncio.get_running_loop()
if os.geteuid() == 0:
log(
"Running the server with root privileges is not recommended.",
Ansi.LRED,
)
app.state.services.http = aiohttp.ClientSession(
json_serialize=lambda x: orjson.dumps(x).decode(), )
await app.state.services.database.connect()
await app.state.services.redis.initialize()
if app.state.services.datadog is not None:
app.state.services.datadog.start(
flush_in_thread=True,
flush_interval=15,
)
app.state.services.datadog.gauge("gulag.online_players", 0)
await app.state.services.run_sql_migrations()
async with app.state.services.database.connection() as db_conn:
await collections.initialize_ram_caches(db_conn)
await app.bg_loops.initialize_housekeeping_tasks()
def point_of_interest():
"""Leave a pseudo-breakpoint somewhere to ask the user if
they could pls submit their stacktrace to cmyui <3."""
# TODO: fix this, circular import thing
#ver_str = f'Running gulag v{glob.version!r} | cmyui_pkg v{cmyui.__version__}'
#printc(ver_str, Ansi.LBLUE)
for fi in inspect.stack()[1:]:
if fi.function == '_run':
# go all the way up to server start func
break
file = Path(fi.filename)
# print line num, index, func name & locals for each frame.
log('[{function}() @ {fname} L{lineno}:{index}] {frame.f_locals}'.
format(**fi._asdict(), fname=file.name))
msg_str = '\n'.join((
"Hey! If you're seeing this, osu! just did something pretty strange,",
"and the gulag devs have left a breakpoint here. We'd really appreciate ",
"if you could screenshot the data above, and send it to cmyui, either via ",
"Discord (cmyui#0425), or by email ([email protected]). Thanks! 😳😳😳"
))
printc(msg_str, Ansi.LRED)
input('To close this menu & unfreeze, simply hit the enter key.')
async def maps_from_sql(self, db_cursor: aiomysql.DictCursor) -> None:
"""Retrieve all maps from sql to populate `self.maps`."""
await db_cursor.execute(
"SELECT map_id, mods, slot FROM tourney_pool_maps WHERE pool_id = %s",
[self.id],
)
async for row in db_cursor:
map_id = row["map_id"]
bmap = await Beatmap.from_bid(map_id)
if not bmap:
# map not found? remove it from the
# pool and log this incident to console.
# NOTE: it's intentional that this removes
# it from not only this pool, but all pools.
# TODO: perhaps discord webhook?
log(f"Removing {map_id} from pool {self.name} (not found).",
Ansi.LRED)
await db_cursor.execute(
"DELETE FROM tourney_pool_maps WHERE map_id = %s",
[map_id],
)
continue
key: tuple[Mods, int] = (Mods(row["mods"]), row["slot"])
self.maps[key] = bmap
async def ensure_local_osu_file(
osu_file_path: Path,
bmap_id: int,
bmap_md5: str,
) -> bool:
"""Ensure we have the latest .osu file locally,
downloading it from the osu!api if required."""
if (not osu_file_path.exists() or
hashlib.md5(osu_file_path.read_bytes()).hexdigest() != bmap_md5):
# need to get the file from the osu!api
if settings.DEBUG:
log(f"Doing osu!api (.osu file) request {bmap_id}", Ansi.LMAGENTA)
url = f"https://old.ppy.sh/osu/{bmap_id}"
async with app.state.services.http.get(url) as resp:
if resp.status != 200:
if 400 <= resp.status < 500:
# client error, report this to cmyui
stacktrace = app.utils.get_appropriate_stacktrace()
await app.state.services.log_strange_occurrence(stacktrace)
return False
osu_file_path.write_bytes(await resp.read())
return True
async def _remove_expired_donation_privileges(interval: int) -> None:
"""Remove donation privileges from users with expired sessions."""
while True:
if glob.app.debug:
log("Removing expired donation privileges.", Ansi.LMAGENTA)
expired_donors = await glob.db.fetchall(
"SELECT id FROM users "
"WHERE donor_end <= UNIX_TIMESTAMP() "
"AND priv & 48", # 48 = Supporter | Premium
_dict=False,
)
for expired_donor_id in expired_donors:
p = await glob.players.from_cache_or_sql(id=expired_donor_id)
# TODO: perhaps make a `revoke_donor` method?
await p.remove_privs(Privileges.DONATOR)
await glob.db.execute(
"UPDATE users SET donor_end = 0 WHERE id = %s",
[p.id],
)
if p.online:
p.enqueue(
packets.notification("Your supporter status has expired."))
log(f"{p}'s supporter status has expired.", Ansi.LMAGENTA)
await asyncio.sleep(interval)
def _download_achievement_images_osu(achievements_path: Path) -> bool:
"""Download all used achievement images (one by one, from osu!)."""
achs = []
for res in ('', '@2x'):
for gm in ('osu', 'taiko', 'fruits', 'mania'):
# only osu!std has 9 & 10 star pass/fc medals.
for n in range(1, 1 + (10 if gm == 'osu' else 8)):
achs.append(f'{gm}-skill-pass-{n}{res}.png')
achs.append(f'{gm}-skill-fc-{n}{res}.png')
for n in (500, 750, 1000, 2000):
achs.append(f'osu-combo-{n}{res}.png')
log('Downloading achievement images from osu!.', Ansi.LCYAN)
for ach in achs:
r = requests.get(f'https://assets.ppy.sh/medals/client/{ach}')
if r.status_code != 200:
return False
log(f'Saving achievement: {ach}', Ansi.LCYAN)
(achievements_path / ach).write_bytes(r.content)
return True
async def ensure_local_osu_file(
osu_file_path: Path,
bmap_id: int, bmap_md5: str
) -> bool:
"""Ensure we have the latest .osu file locally,
downloading it from the osu!api if required."""
if (
not osu_file_path.exists() or
hashlib.md5(osu_file_path.read_bytes()).hexdigest() != bmap_md5
):
# need to get the file from the osu!api
if glob.app.debug:
log(f'Doing osu!api (.osu file) request {bmap_id}', Ansi.LMAGENTA)
url = f'https://old.ppy.sh/osu/{bmap_id}'
async with glob.http.get(url) as r:
if not r or r.status != 200:
# temporary logging, not sure how possible this is
stacktrace = utils.misc.get_appropriate_stacktrace()
await utils.misc.log_strange_occurrence(stacktrace)
return False
osu_file_path.write_bytes(await r.read())
return True
async def maps_from_sql(self, db_conn: databases.core.Connection) -> None:
"""Retrieve all maps from sql to populate `self.maps`."""
for row in await db_conn.fetch_all(
"SELECT map_id, mods, slot FROM tourney_pool_maps WHERE pool_id = :pool_id",
{"pool_id": self.id},
):
map_id = row["map_id"]
bmap = await Beatmap.from_bid(map_id)
if not bmap:
# map not found? remove it from the
# pool and log this incident to console.
# NOTE: it's intentional that this removes
# it from not only this pool, but all pools.
# TODO: perhaps discord webhook?
log(f"Removing {map_id} from pool {self.name} (not found).",
Ansi.LRED)
await db_conn.execute(
"DELETE FROM tourney_pool_maps WHERE map_id = :map_id",
{"map_id": map_id},
)
continue
key: tuple[Mods, int] = (Mods(row["mods"]), row["slot"])
self.maps[key] = bmap
async def auth_osu_api(bot) -> None:
try:
bot.osu = OssapiV2(client_id=config.osu.get("id"), client_secret=config.osu.get("secret"))
log("Authorized with the osu!api!", Ansi.LGREEN)
except:
bot.unload_extension("cogs.osu")
log("Failed to authorize with the osu!api! (Unloaded cogs.osu!)", Ansi.LRED)
async def dispatch(
self,
request: Request,
call_next: RequestResponseEndpoint,
) -> Response:
start_time = time.perf_counter_ns()
response = await call_next(request)
end_time = time.perf_counter_ns()
time_elapsed = end_time - start_time
# TODO: add metric to datadog
col = (
Ansi.LGREEN
if 200 <= response.status_code < 300
else Ansi.LYELLOW
if 300 <= response.status_code < 400
else Ansi.LRED
)
url = f"{request.headers['host']}{request['path']}"
log(f"[{request.method}] {response.status_code} {url}", col, end=" | ")
printc(f"Request took: {magnitude_fmt_time(time_elapsed)}", Ansi.LBLUE)
response.headers["process-time"] = str(round(time_elapsed) / 1e6)
return response
async def fetch_geoloc_web(ip: IPAddress) -> dict[str, Union[str, float]]:
"""Fetch geolocation data based on ip (using ip-api)."""
if not glob.has_internet: # requires internet connection
return
url = f'http://ip-api.com/line/{ip}'
async with glob.http.get(url) as resp:
if not resp or resp.status != 200:
log('Failed to get geoloc data: request failed.', Ansi.LRED)
return
status, *lines = (await resp.text()).split('\n')
if status != 'success':
err_msg = lines[0]
if err_msg == 'invalid query':
err_msg += f' ({url})'
log(f'Failed to get geoloc data: {err_msg}.', Ansi.LRED)
return
acronym = lines[1].lower()
return {
'latitude': float(lines[6]),
'longitude': float(lines[7]),
'country': {
'acronym': acronym,
'numeric': country_codes[acronym]
}
}
async def _remove_expired_donation_privileges(interval: int) -> None:
"""Remove donation privileges from users with expired sessions."""
while True:
if settings.DEBUG:
log("Removing expired donation privileges.", Ansi.LMAGENTA)
expired_donors = await app.state.services.database.fetch_all(
"SELECT id FROM users "
"WHERE donor_end <= UNIX_TIMESTAMP() "
"AND priv & 48", # 48 = Supporter | Premium
)
for expired_donor in expired_donors:
p = await app.state.sessions.players.from_cache_or_sql(
id=expired_donor["id"], )
assert p is not None
# TODO: perhaps make a `revoke_donor` method?
await p.remove_privs(Privileges.DONATOR)
await app.state.services.database.execute(
"UPDATE users SET donor_end = 0 WHERE id = :id",
{"id": p.id},
)
if p.online:
p.enqueue(
app.packets.notification(
"Your supporter status has expired."), )
log(f"{p}'s supporter status has expired.", Ansi.LMAGENTA)
await asyncio.sleep(interval)
async def maps_from_sql(self) -> None:
"""Retrieve all maps from sql to populate `self.maps`."""
query = ('SELECT map_id, mods, slot '
'FROM tourney_pool_maps '
'WHERE pool_id = %s')
for row in await glob.db.fetchall(query, [self.id]):
map_id = row['map_id']
bmap = await Beatmap.from_bid(map_id)
if not bmap:
# map not found? remove it from the
# pool and log this incident to console.
# NOTE: it's intentional that this removes
# it from not only this pool, but all pools.
# TODO: perhaps discord webhook?
log(f'Removing {map_id} from pool {self.name} (not found).', Ansi.LRED)
await glob.db.execute(
'DELETE FROM tourney_pool_maps '
'WHERE map_id = %s',
[map_id]
)
continue
key = (Mods(row['mods']), row['slot'])
self.maps[key] = bmap
async def mysql_connect(bot, cred: dict) -> None:
try:
bot.db = AsyncSQLPool()
await bot.db.connect(cred)
log("Connected to MySQL!", Ansi.LGREEN)
except:
log("Failed to connect to MySQL!", Ansi.LRED)
exit(1) # NOTE: Moé loses a lot of functionality without mysql; stop execution
def append(self, p: Player) -> None:
"""Append `p` to the list."""
if p in self:
if settings.DEBUG:
log(f"{p} double-added to global player list?")
return
super().append(p)
def remove(self, p: Player) -> None:
"""Remove `p` from the list."""
if p not in self:
if settings.DEBUG:
log(f"{p} removed from player list when not online?")
return
super().remove(p)
def remove(self, p: Player) -> None:
"""Remove `p` from the list."""
if p not in self:
if glob.app.debug:
log(f'{p} removed from player list when not online?')
return
super().remove(p)
def append(self, p: Player) -> None:
"""Append `p` to the list."""
if p in self:
if glob.app.debug:
log(f'{p} double-added to global player list?')
return
super().append(p)
async def create_client_session(bot) -> None:
try:
bot.request = aiohttp.ClientSession(json_serialize=orjson.dumps)
log("Client Session created!", Ansi.LGREEN)
except:
log("Failed to get the Client Session!", Ansi.LRED)
bot.db.close() # safely close db connection
exit(1) # NOTE: Moé loses a lot of functionality without the client session; stop execution
async def from_api(cls, md5: str) -> Optional['Beatmap']:
api = 'https://old.ppy.sh/api/get_beatmaps'
params = {'k': glob.config.api_key, 'h': md5}
async with glob.web.get(api, params=params) as resp:
if resp.status != 200 or not resp:
return # request failed, map prob doesnt exist
data = await resp.json()
if not data:
return
bmap = data[0] # i hate this idea but o well
self = cls()
self.id = int(bmap['beatmap_id'])
self.sid = int(bmap['beatmapset_id'])
self.md5 = md5
self.bpm = float(bmap['bpm'])
self.cs = float(bmap['diff_size'])
self.ar = float(bmap['diff_approach'])
self.od = float(bmap['diff_overall'])
self.hp = float(bmap['diff_drain'])
self.sr = float(bmap['difficultyrating'])
self.mode = osuModes(int(bmap['mode']))
self.artist = bmap['artist']
self.title = bmap['title']
self.diff = bmap['version']
self.mapper = bmap['creator']
self.status = mapStatuses.from_api(int(bmap['approved']))
self.update = dt.strptime(bmap['last_update'], '%Y-%m-%d %H:%M:%S').timestamp()
self.nc = time.time()
e = await glob.db.fetchrow('SELECT frozen, status, `update` FROM maps WHERE id = %s', [self.id])
if e:
if self.update > e['update']:
if e['frozen'] and self.status != e['status']:
self.status = e['status']
self.frozen = e['frozen'] == 1
self.lb = None # status has changed, lets reset lb cache in case
await self.save()
else:
pass
else:
self.frozen = False # don't freeze by default, we can override if someone manually edits the map status
await self.save()
glob.cache['maps'][md5] = self # cache the map now we have it from api & saved in sql
await cls.cache_set(self.sid)
log(f'Retrieved Set ID {self.sid} from osu!api', Ansi.LCYAN)
return self
def remove(self, m: Match) -> None:
"""Remove `m` from the list."""
for i, _m in enumerate(self):
if m is _m:
self[i] = None
break
if glob.app.debug:
log(f'{m} removed from matches list.')
def remove(self, m: Match) -> None:
"""Remove `m` from the list."""
for i, _m in enumerate(self):
if m is _m:
self[i] = None
break
if settings.DEBUG:
log(f"{m} removed from matches list.")
async def prepare(self, db_conn: databases.core.Connection) -> None:
"""Fetch data from sql & return; preparing to run the server."""
log("Fetching clans from sql.", Ansi.LCYAN)
for row in await db_conn.fetch_all("SELECT * FROM clans"):
row = dict(row) # make a mutable copy
row["owner_id"] = row.pop("owner")
clan = Clan(**row)
await clan.members_from_sql(db_conn)
self.append(clan)
async def send_pm(user: Player, p: bytes) -> None:
d = reader.handle_packet(p, (('msg', osuTypes.message),))
msg = d['msg'].msg
tarname = d['msg'].tarname
if not (target := await glob.players.get(name=tarname)):
log(f'{user.name} tried to send message to offline user {tarname}', Ansi.LRED)
return
async def friend_remove(user: Player, p: bytes) -> None:
tar = (reader.handle_packet(p, (('uid', osuTypes.i32),)))['uid']
if tar not in user.friends:
return
user.friends.remove(tar)
await glob.db.execute('DELETE FROM friends WHERE user1 = %s AND user2 = %s', [user.id, tar])
log(f"{user.name} removed UID {tar} from their friends list.", Ansi.LCYAN)