from common.files import fileBuffer, fileLocks
from common.web import schiavo
try:
with open("version") as f:
VERSION = f.read().strip()
except:
VERSION = "Unknown"
ACHIEVEMENTS_VERSION = 1
DATADOG_PREFIX = "lets"
BOT_NAME = "Charlotte"
db = None
redis = None
conf = None
application = None
pool = None
pascoa = {}
busyThreads = 0
debug = False
sentry = False
# Cache and objects
fLocks = fileLocks.fileLocks()
userStatsCache = userStatsCache.userStatsCache()
personalBestCache = personalBestCache.personalBestCache()
fileBuffers = fileBuffer.buffersList()
dog = datadogClient.datadogClient()
schiavo = schiavo.schiavo()
achievementClasses = {}
# Set up datadog
try:
if generalUtils.stringToBool(
glob.conf.config["datadog"]["enable"]):
glob.dog = datadogClient.datadogClient(
glob.conf.config["datadog"]["apikey"],
glob.conf.config["datadog"]["appkey"],
[
datadogClient.periodicCheck(
"online_users", lambda: len(glob.tokens.tokens)),
datadogClient.periodicCheck(
"multiplayer_matches",
lambda: len(glob.matches.matches)),
#datadogClient.periodicCheck("ram_clients", lambda: generalUtils.getTotalSize(glob.tokens)),
#datadogClient.periodicCheck("ram_matches", lambda: generalUtils.getTotalSize(glob.matches)),
#datadogClient.periodicCheck("ram_channels", lambda: generalUtils.getTotalSize(glob.channels)),
#datadogClient.periodicCheck("ram_file_buffers", lambda: generalUtils.getTotalSize(glob.fileBuffers)),
#datadogClient.periodicCheck("ram_file_locks", lambda: generalUtils.getTotalSize(glob.fLocks)),
#datadogClient.periodicCheck("ram_datadog", lambda: generalUtils.getTotalSize(glob.datadogClient)),
#datadogClient.periodicCheck("ram_verified_cache", lambda: generalUtils.getTotalSize(glob.verifiedCache)),
#datadogClient.periodicCheck("ram_irc", lambda: generalUtils.getTotalSize(glob.ircServer)),
#datadogClient.periodicCheck("ram_tornado", lambda: generalUtils.getTotalSize(glob.application)),
#datadogClient.periodicCheck("ram_db", lambda: generalUtils.getTotalSize(glob.db)),
])
else:
consoleHelper.printColored(
"[!] Warning! Datadog stats tracking is disabled!",
bcolors.YELLOW)
except:
glob.application.sentry_client = AsyncSentryClient(
glob.conf.config["sentry"]["dsn"], release=glob.VERSION)
else:
consoleHelper.printColored(
"[!] Warning! Sentry logging is disabled!", bcolors.YELLOW)
except:
consoleHelper.printColored(
"[!] Error while starting Sentry client! Please check your config.ini and run the server again",
bcolors.RED)
# Set up Datadog
try:
if generalUtils.stringToBool(
glob.conf.config["datadog"]["enable"]):
glob.dog = datadogClient.datadogClient(
glob.conf.config["datadog"]["apikey"],
glob.conf.config["datadog"]["appkey"])
else:
consoleHelper.printColored(
"[!] Warning! Datadog stats tracking is disabled!",
bcolors.YELLOW)
except:
consoleHelper.printColored(
"[!] Error while starting Datadog client! Please check your config.ini and run the server again",
bcolors.RED)
# Connect to pubsub channels
pubSub.listener(glob.redis, {
"lets:beatmap_updates": beatmapUpdateHandler.handler(),
}).start()
# Set up sentry
try:
if glob.conf.sentry_enabled:
glob.application.sentry_client = AsyncSentryClient(glob.conf["SENTRY_DSN"], release=glob.VERSION)
else:
consoleHelper.printColored("[!] Warning! Sentry logging is disabled!", bcolors.YELLOW)
except:
consoleHelper.printColored("[!] Error while starting Sentry client! Please check your config.ini and run the server again", bcolors.RED)
# Set up Datadog
try:
if glob.conf.datadog_enabled:
glob.dog = datadogClient.datadogClient(
glob.conf["DATADOG_API_KEY"],
glob.conf["DATADOG_APP_KEY"],
constant_tags=["worker:{}".format(glob.serverPort)]
)
else:
consoleHelper.printColored("[!] Warning! Datadog stats tracking is disabled!", bcolors.YELLOW)
except:
consoleHelper.printColored("[!] Error while starting Datadog client! Please check your config.ini and run the server again", bcolors.RED)
# Connect to pubsub channels
pubSub.listener(glob.redis, {
"lets:beatmap_updates": beatmapUpdateHandler.handler(),
"lets:reload_aql": lambda x: x == b"reload" and glob.aqlThresholds.reload(),
}).start()
# Check debug mods
if glob.conf["DEBUG"]:
def main():
parser = argparse.ArgumentParser(
description=consoleHelper.ASCII +
"\n\nLatest Essential Tatoe Server v{}\nBy The Ripple Team".format(
glob.VERSION),
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument("-p",
"--port",
help="Run on a specific port (bypasses config.ini)",
required=False)
parser.add_argument(
"-s",
"--stats-port",
help="Run prometheus on a specific port (bypasses config.ini)",
required=False)
parser.add_argument("-q",
"--quiet",
help="Log less stuff during startup",
required=False,
default=False,
action="store_true")
cli_args = parser.parse_args()
# AGPL license agreement
try:
agpl.check_license("ripple", "LETS")
except agpl.LicenseError as e:
logging.error(str(e))
sys.exit(1)
try:
if not cli_args.quiet:
consoleHelper.printServerStartHeader(True)
def loudLog(s, f=logging.info):
if not cli_args.quiet:
f(s)
# Read config
loudLog("Reading config file... ")
glob.conf = Config()
# Create data/oppai maps folder if needed
loudLog("Checking folders... ")
paths = (".data", glob.conf["BEATMAPS_FOLDER"],
glob.conf["SCREENSHOTS_FOLDER"],
glob.conf["FAILED_REPLAYS_FOLDER"],
glob.conf["REPLAYS_FOLDER"])
for i in paths:
if not os.path.exists(i):
os.makedirs(i, 0o770)
# Connect to db
try:
loudLog("Connecting to MySQL database")
glob.db = dbConnector.db(host=glob.conf["DB_HOST"],
port=glob.conf["DB_PORT"],
user=glob.conf["DB_USERNAME"],
password=glob.conf["DB_PASSWORD"],
database=glob.conf["DB_NAME"],
autocommit=True,
charset="utf8")
glob.db.fetch("SELECT 1")
except:
# Exception while connecting to db
logging.error(
"Error while connection to database. Please check your config.ini and run the server again"
)
raise
# Connect to redis
try:
loudLog("Connecting to redis")
glob.redis = redis.Redis(glob.conf["REDIS_HOST"],
glob.conf["REDIS_PORT"],
glob.conf["REDIS_DATABASE"],
glob.conf["REDIS_PASSWORD"])
glob.redis.ping()
except:
# Exception while connecting to db
logging.error(
"Error while connection to redis. Please check your config.ini and run the server again"
)
raise
# Empty redis cache
try:
glob.redis.eval(
"return redis.call('del', unpack(redis.call('keys', ARGV[1])))",
0, "lets:*")
except redis.exceptions.ResponseError:
# Script returns error if there are no keys starting with peppy:*
pass
# Save lets version in redis
glob.redis.set("lets:version", glob.VERSION)
# Create threads pool
try:
loudLog("Creating threads pool")
glob.pool = ThreadPool(glob.conf["THREADS"])
except:
logging.error(
"Error while creating threads pool. Please check your config.ini and run the server again"
)
raise
# Check osuapi
if not glob.conf["OSU_API_ENABLE"]:
logging.warning(
"osu!api features are disabled. If you don't have a "
"valid beatmaps table, all beatmaps will show as unranked")
if glob.conf["BEATMAP_CACHE_EXPIRE"] > 0:
logging.warning(
"IMPORTANT! Your beatmapcacheexpire in config.ini is > 0 and osu!api "
"features are disabled.\nWe do not recommend this, because too old "
"beatmaps will be shown as unranked.\nSet beatmapcacheexpire to 0 to "
"disable beatmap latest update check and fix that issue.")
# Load achievements
#loudLog("Loading achievements")
#try:
# secret.achievements.utils.load_achievements()
#except:
# logging.error("Error while loading achievements")
# raise
glob.ACHIEVEMENTS_VERSION = "0.0.0"
# Set achievements version
glob.redis.set("lets:achievements_version", glob.ACHIEVEMENTS_VERSION)
loudLog("Achievements version is {}".format(glob.ACHIEVEMENTS_VERSION))
# Check if s3 is enabled
if not glob.conf.s3_enabled:
loudLog("S3 is disabled!", logging.warning)
else:
c = glob.db.fetch(
"SELECT COUNT(*) AS c FROM s3_replay_buckets WHERE max_score_id IS NULL"
)["c"]
if c != 1:
logging.error(
"There must be only one bucket flagged as WRITE bucket! You have {}."
.format(c), )
sys.exit()
# Discord
if glob.conf.schiavo_enabled:
glob.schiavo = schiavo.schiavo(glob.conf["SCHIAVO_URL"],
"**lets**")
else:
logging.warning("Schiavo logging is disabled!")
# Server port
try:
if cli_args.port:
loudLog(
"Running on port {}, bypassing config.ini".format(
cli_args.port), logging.warning)
glob.serverPort = int(cli_args.port)
else:
glob.serverPort = glob.conf["HTTP_PORT"]
except:
logging.error(
"Invalid server port! Please check your config.ini and run the server again"
)
raise
# Prometheus port
try:
if cli_args.stats_port:
loudLog(
"Running stats exporter on port {}, bypassing config.ini".
format(cli_args.stats_port), logging.warning)
glob.statsPort = int(cli_args.stats_port)
elif glob.conf["PROMETHEUS_PORT"]:
glob.statsPort = int(glob.conf["PROMETHEUS_PORT"])
except:
logging.error(
"Invalid stats port! Please check your config.ini and run the server again"
)
raise
# Make app
glob.application = make_app()
# Set up sentry
if glob.conf.sentry_enabled:
glob.application.sentry_client = AsyncSentryClient(
glob.conf["SENTRY_DSN"], release=glob.VERSION)
else:
loudLog("Sentry logging is disabled!", logging.warning)
# Set up Datadog
if glob.conf.datadog_enabled:
glob.dog = datadogClient.datadogClient(
glob.conf["DATADOG_API_KEY"],
glob.conf["DATADOG_APP_KEY"],
constant_tags=["worker:{}".format(glob.serverPort)])
else:
glob.dog = datadogClient.datadogClient()
loudLog("Datadog stats tracking is disabled!", logging.warning)
# Connect to pubsub channels
t = pubSub.listener(
glob.redis, {
"lets:beatmap_updates":
beatmapUpdateHandler.handler(),
"lets:reload_aql":
lambda x: x == b"reload" and glob.aqlThresholds.reload(),
})
t.setDaemon(True)
t.start()
# Check debug mods
if glob.conf["DEBUG"]:
logging.warning("Server running in debug mode.")
# Close main thread db connection as we don't need it anymore
glob.threadScope.dbClose()
# Server start message and console output
logging.info("L.E.T.S. is listening for clients on {}:{}...".format(
glob.conf["HTTP_HOST"], glob.serverPort))
# log.discord("bunker", "Server started!")
# Start Tornado
def term(_, __):
tornado.ioloop.IOLoop.instance().add_callback_from_signal(
lambda: tornado.ioloop.IOLoop.instance().stop())
signal.signal(signal.SIGINT, term)
signal.signal(signal.SIGTERM, term)
if glob.statsPort is not None:
logging.info("Stats exporter listening on 0.0.0.0:{}".format(
glob.statsPort))
prometheus_client.start_http_server(glob.statsPort, addr="0.0.0.0")
glob.application.listen(glob.serverPort,
address=glob.conf["HTTP_HOST"])
tornado.ioloop.IOLoop.instance().start()
logging.debug("IOLoop stopped")
finally:
# Perform some clean up
logging.info("Disposing server")
glob.fileBuffers.flushAll()
if glob.redis.connection_pool is not None:
glob.redis.connection_pool.disconnect()
# TODO: properly dispose mysql connections
if glob.pool is not None:
# Close db conn in each thread
glob.pool.imap(lambda *_: glob.threadScope.dbClose(),
[None] * glob.conf["THREADS"],
chunksize=1)
# Wait for everything else to finish (should always terminate immediately)
glob.pool.close()
glob.pool.join()
logging.info("Goodbye!")
def main() -> int:
# AGPL license agreement
try:
agpl.check_license("ripple", "LETS")
except agpl.LicenseError as e:
print(str(e))
return 1
try:
consoleHelper.printServerStartHeader(True)
# Read config
consoleHelper.printNoNl("> Reading config file... ")
glob.conf = config.config("config.ini")
if glob.conf.default:
# We have generated a default config.ini, quit server
consoleHelper.printWarning()
consoleHelper.printColored(
"[!] config.ini not found. A default one has been generated.",
bcolors.YELLOW)
consoleHelper.printColored(
"[!] Please edit your config.ini and run the server again.",
bcolors.YELLOW)
return 1
# If we haven't generated a default config.ini, check if it's valid
if not glob.conf.checkConfig():
consoleHelper.printError()
consoleHelper.printColored(
"[!] Invalid config.ini. Please configure it properly",
bcolors.RED)
consoleHelper.printColored(
"[!] Delete your config.ini to generate a default one",
bcolors.RED)
return 1
else:
consoleHelper.printDone()
# Read additional config file
consoleHelper.printNoNl("> Loading additional config file... ")
try:
if not os.path.isfile(glob.conf.config["custom"]["config"]):
consoleHelper.printWarning()
consoleHelper.printColored(
"[!] Missing config file at {}; A default one has been generated at this location."
.format(glob.conf.config["custom"]["config"]),
bcolors.YELLOW)
shutil.copy("common/default_config.json",
glob.conf.config["custom"]["config"])
with open(glob.conf.config["custom"]["config"], "r") as f:
glob.conf.extra = json.load(f)
consoleHelper.printDone()
except:
consoleHelper.printWarning()
consoleHelper.printColored(
"[!] Unable to load custom config at {}".format(
glob.conf.config["custom"]["config"]), bcolors.RED)
return 1
# Create data/oppai maps folder if needed
consoleHelper.printNoNl("> Checking folders... ")
paths = [
".data", ".data/oppai", ".data/catch_the_pp",
glob.conf.config["server"]["replayspath"],
"{}_relax".format(glob.conf.config["server"]["replayspath"]),
glob.conf.config["server"]["beatmapspath"],
glob.conf.config["server"]["screenshotspath"]
]
for i in paths:
if not os.path.exists(i):
os.makedirs(i, 0o770)
consoleHelper.printDone()
# Connect to db
try:
consoleHelper.printNoNl("> Connecting to MySQL database... ")
glob.db = dbConnector.db(glob.conf.config["db"]["host"],
glob.conf.config["db"]["username"],
glob.conf.config["db"]["password"],
glob.conf.config["db"]["database"],
int(glob.conf.config["db"]["workers"]))
consoleHelper.printNoNl(" ")
consoleHelper.printDone()
except:
# Exception while connecting to db
consoleHelper.printError()
consoleHelper.printColored(
"[!] Error while connection to database. Please check your config.ini and run the server again",
bcolors.RED)
raise
# Connect to redis
try:
consoleHelper.printNoNl("> Connecting to redis... ")
glob.redis = redis.Redis(glob.conf.config["redis"]["host"],
glob.conf.config["redis"]["port"],
glob.conf.config["redis"]["database"],
glob.conf.config["redis"]["password"])
glob.redis.ping()
consoleHelper.printNoNl(" ")
consoleHelper.printDone()
except:
# Exception while connecting to db
consoleHelper.printError()
consoleHelper.printColored(
"[!] Error while connection to redis. Please check your config.ini and run the server again",
bcolors.RED)
raise
# Empty redis cache
#TODO: do we need this?
try:
glob.redis.eval(
"return redis.call('del', unpack(redis.call('keys', ARGV[1])))",
0, "lets:*")
except redis.exceptions.ResponseError:
# Script returns error if there are no keys starting with peppy:*
pass
# Save lets version in redis
glob.redis.set("lets:version", glob.VERSION)
# Create threads pool
try:
consoleHelper.printNoNl("> Creating threads pool... ")
glob.pool = ThreadPool(int(glob.conf.config["server"]["threads"]))
consoleHelper.printDone()
except:
consoleHelper.printError()
consoleHelper.printColored(
"[!] Error while creating threads pool. Please check your config.ini and run the server again",
bcolors.RED)
# Load achievements
consoleHelper.printNoNl("> Loading achievements... ")
try:
achievements = glob.db.fetchAll("SELECT * FROM achievements")
for achievement in achievements:
condition = eval(
f"lambda score, mode_vn, stats: {achievement.pop('cond')}")
glob.achievements.append(
Achievement(_id=achievement['id'],
file=achievement['icon'],
name=achievement['name'],
desc=achievement['description'],
cond=condition))
except Exception as e:
consoleHelper.printError()
consoleHelper.printColored(
"[!] Error while loading achievements! ({})".format(
traceback.format_exc()),
bcolors.RED,
)
return 1
consoleHelper.printDone()
# Set achievements version
glob.redis.set("lets:achievements_version", glob.ACHIEVEMENTS_VERSION)
consoleHelper.printColored(
"Achievements version is {}".format(glob.ACHIEVEMENTS_VERSION),
bcolors.YELLOW)
# Print disallowed mods into console (Used to also assign it into variable but has been moved elsewhere)
unranked_mods = [
key for key, value in glob.conf.extra["common"]
["rankable-mods"].items() if not value
]
consoleHelper.printColored(
"Unranked mods: {}".format(", ".join(unranked_mods)),
bcolors.YELLOW)
# Print allowed beatmap rank statuses
allowed_beatmap_rank = [
key for key, value in glob.conf.extra["lets"]
["allowed-beatmap-rankstatus"].items() if value
]
consoleHelper.printColored(
"Allowed beatmap rank statuses: {}".format(
", ".join(allowed_beatmap_rank)), bcolors.YELLOW)
# Make array of bools to respective rank id's
glob.conf.extra["_allowed_beatmap_rank"] = [
getattr(rankedStatuses, key) for key in allowed_beatmap_rank
] # Store the allowed beatmap rank id's into glob
# Discord
if generalUtils.stringToBool(glob.conf.config["discord"]["enable"]):
glob.schiavo = schiavo.schiavo(
glob.conf.config["discord"]["boturl"], "**lets**")
else:
consoleHelper.printColored(
"[!] Warning! Discord logging is disabled!", bcolors.YELLOW)
# Check debug mods
glob.debug = generalUtils.stringToBool(
glob.conf.config["server"]["debug"])
if glob.debug:
consoleHelper.printColored(
"[!] Warning! Server running in debug mode!", bcolors.YELLOW)
# Server port
try:
serverPort = int(glob.conf.config["server"]["port"])
except:
consoleHelper.printColored(
"[!] Invalid server port! Please check your config.ini and run the server again",
bcolors.RED)
# Make app
glob.application = make_app()
# Set up sentry
try:
glob.sentry = generalUtils.stringToBool(
glob.conf.config["sentry"]["enable"])
if glob.sentry:
glob.application.sentry_client = AsyncSentryClient(
glob.conf.config["sentry"]["dsn"], release=glob.VERSION)
else:
consoleHelper.printColored(
"[!] Warning! Sentry logging is disabled!", bcolors.YELLOW)
except:
consoleHelper.printColored(
"[!] Error while starting Sentry client! Please check your config.ini and run the server again",
bcolors.RED)
# Set up Datadog
try:
if generalUtils.stringToBool(
glob.conf.config["datadog"]["enable"]):
glob.dog = datadogClient.datadogClient(
glob.conf.config["datadog"]["apikey"],
glob.conf.config["datadog"]["appkey"])
else:
consoleHelper.printColored(
"[!] Warning! Datadog stats tracking is disabled!",
bcolors.YELLOW)
except:
consoleHelper.printColored(
"[!] Error while starting Datadog client! Please check your config.ini and run the server again",
bcolors.RED)
# Connect to pubsub channels
pubSub.listener(glob.redis, {
"lets:beatmap_updates": beatmapUpdateHandler.handler(),
}).start()
# Prometheus port
statsPort = None
try:
if glob.conf.config["prometheus"]["port"]:
statsPort = int(glob.conf.config["prometheus"]["port"])
except:
consoleHelper.printColored(
"Invalid stats port! Please check your config.ini and run the server again",
bcolors.YELLOW)
raise
if statsPort:
consoleHelper.printColored(
"Stats exporter listening on localhost:{}".format(statsPort),
bcolors.GREEN)
prometheus_client.start_http_server(statsPort, addr="127.0.0.1")
# Server start message and console output
consoleHelper.printColored(
"> L.E.T.S. is listening for clients on {}:{}...".format(
glob.conf.config["server"]["host"], serverPort), bcolors.GREEN)
# Start Tornado
glob.application.listen(serverPort,
address=glob.conf.config["server"]["host"])
tornado.ioloop.IOLoop.instance().start()
finally:
# Perform some clean up
print("> Disposing server... ")
glob.fileBuffers.flushAll()
consoleHelper.printColored("Goodbye!", bcolors.GREEN)
return 0
# Set up sentry
try:
glob.sentry = generalUtils.stringToBool(glob.conf.config["sentry"]["enable"])
if glob.sentry:
glob.application.sentry_client = AsyncSentryClient(glob.conf.config["sentry"]["dsn"], release=glob.VERSION)
else:
consoleHelper.printColored("[!] Warning! Sentry logging is disabled!", bcolors.YELLOW)
except:
consoleHelper.printColored("[!] Error while starting Sentry client! Please check your config.ini and run the server again", bcolors.RED)
# Set up Datadog
try:
if generalUtils.stringToBool(glob.conf.config["datadog"]["enable"]):
glob.dog = datadogClient.datadogClient(
glob.conf.config["datadog"]["apikey"],
glob.conf.config["datadog"]["appkey"],
constant_tags=["worker:{}".format(glob.serverPort)]
)
else:
consoleHelper.printColored("[!] Warning! Datadog stats tracking is disabled!", bcolors.YELLOW)
except:
consoleHelper.printColored("[!] Error while starting Datadog client! Please check your config.ini and run the server again", bcolors.RED)
# Connect to pubsub channels
pubSub.listener(glob.redis, {
"lets:beatmap_updates": beatmapUpdateHandler.handler(),
"lets:reload_aql": lambda x: x == b"reload" and glob.aqlThresholds.reload(),
}).start()
# Server start message and console output
consoleHelper.printColored("> L.E.T.S. is listening for clients on {}:{}...".format(
