def load_config(args):
# Load config
logging.info("Reading config file")
glob.conf = config.config("config.ini")
# Read additional config file
logging.info("Reading additional config file")
with open(glob.conf.config["custom"]["config"], "r") as f:
logging.info("Add-on conf = {}".format(
glob.conf.config["custom"]["config"]))
glob.conf.extra = json.load(f)
logging.info("Connecting to MySQL db")
glob.db = dbConnector.db(glob.conf.config["db"]["host"],
glob.conf.config["db"]["username"],
glob.conf.config["db"]["password"],
glob.conf.config["db"]["database"],
max(1, MAX_WORKERS))
# Set verbose
glob.debug = args.verbose
# Disable MySQL db warnings (it spams 'Unsafe statement written to the binary log using statement...'
# because we use UPDATE with LIMIT 1 when updating performance points after recalculation
warnings.filterwarnings("ignore", category=MySQLdb.Warning)
def main():
# CLI stuff
parser = argparse.ArgumentParser(
description="pp recalc tool for ripple, new version.")
recalc_group = parser.add_mutually_exclusive_group(required=False)
recalc_group.add_argument("-r",
"--recalc",
help="calculates pp for all high scores",
required=False,
action="store_true")
recalc_group.add_argument("-z",
"--zero",
help="calculates pp for 0 pp high scores",
required=False,
action="store_true")
recalc_group.add_argument(
"-i",
"--id",
help="calculates pp for the score with this score_id",
required=False)
recalc_group.add_argument(
"-m",
"--mods",
help="calculates pp for high scores with these mods (flags)",
required=False)
recalc_group.add_argument(
"-g",
"--gamemode",
help=
"calculates pp for scores played on this game mode (std:0, taiko:1, ctb:2, mania:3)",
required=False)
recalc_group.add_argument("-l",
"--loved",
help="calculate pp for loved maps",
required=False)
recalc_group.add_argument(
"-u",
"--userid",
help="calculates pp for high scores set by a specific user (user_id)",
required=False)
recalc_group.add_argument(
"-b",
"--beatmapid",
help=
"calculates pp for high scores played on a specific beatmap (beatmap_id)",
required=False)
recalc_group.add_argument(
"-fhd",
"--fixstdhd",
help=
"calculates pp for std hd high scores (14/05/2018 pp algorithm changes)",
required=False,
action="store_true")
parser.add_argument("-w",
"--workers",
help="number of workers. {} by default. Max {}".format(
MAX_WORKERS // 2, MAX_WORKERS),
required=False)
parser.add_argument("-cs",
"--chunksize",
help="score chunks size",
required=False)
parser.add_argument("-v",
"--verbose",
help="verbose/debug mode",
required=False,
action="store_true")
args = parser.parse_args()
# Logging
progressbar.streams.wrap_stderr()
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
logging.info("Running under {}".format("UNIX" if UNIX else "WIN32"))
# Load config
logging.info("Reading config file")
glob.conf = config.config("config.ini")
# Get workers from arguments if set
workers_number = MAX_WORKERS // 2
if args.workers is not None:
workers_number = int(args.workers)
# Get chunk size from arguments if set
chunk_size = None
if args.chunksize is not None:
chunk_size = int(args.chunksize)
# Disable MySQL db warnings (it spams 'Unsafe statement written to the binary log using statement...'
# because we use UPDATE with LIMIT 1 when updating performance points after recalculation
warnings.filterwarnings("ignore", category=MySQLdb.Warning)
# Connect to MySQL
logging.info("Connecting to MySQL db")
glob.db = dbConnector.db(glob.conf.config["db"]["host"],
glob.conf.config["db"]["username"],
glob.conf.config["db"]["password"],
glob.conf.config["db"]["database"],
max(workers_number, MAX_WORKERS))
# Set verbose
glob.debug = args.verbose
# Get recalculator
recalculators_gen = {
"zero":
lambda: SimpleRecalculator(("scores_auto.completed = 3", "pp = 0")),
"recalc":
lambda: SimpleRecalculator(("scores_auto.completed = 3", "pp > 750")),
"mods":
lambda: SimpleRecalculator(
("scores_auto.completed = 3", "mods & %s > 0"), (args.mods, )),
"id":
lambda: SimpleRecalculator(("scores_auto.id = %s", ), (args.id, )),
"gamemode":
lambda: SimpleRecalculator((
"scores_auto.completed = 3",
"scores_auto.play_mode = %s",
), (args.gamemode, )),
"loved":
lambda: SimpleRecalculator(
("scores_auto.completed = 3", "beatmaps.ranked = 5")),
"userid":
lambda: SimpleRecalculator((
"scores_auto.completed = 3",
"scores_auto.userid = %s",
), (args.userid, )),
"beatmapid":
lambda: SimpleRecalculator((
"scores_auto.completed = 3",
"beatmaps.beatmap_id = %s",
), (args.beatmapid, )),
"fixstdhd":
lambda: SimpleRecalculator(
("scores_auto.completed = 3", "scores_auto.play_mode = 0",
"scores_auto.mods & 8 > 0"))
}
recalculator = None
for k, v in vars(args).items():
if v is not None and ((type(v) is bool and v) or type(v) is not bool):
if k in recalculators_gen:
recalculator = recalculators_gen[k]()
break
# Execute mass recalc
if recalculator is not None:
mass_recalc(recalculator, workers_number, chunk_size)
else:
logging.warning("No recalc option specified")
parser.print_help()
import argparse
import json
import subprocess
from helpers.config import config
from helpers.log import log
if __name__ == '__main__':
log('Pipeline invoked...')
cfg = config('./config.json')
parser = argparse.ArgumentParser()
parser.add_argument('--preprocess', type=bool, default=cfg['pipeline']['preprocess'])
parser.add_argument('--train', type=bool, default=cfg['pipeline']['train'])
parser.add_argument('--explain', type=bool, default=cfg['pipeline']['explain'])
parser.add_argument('--postprocess', type=bool, default=cfg['pipeline']['postprocess'])
args = parser.parse_args()
print(json.dumps(cfg, indent=2))
if args.preprocess:
subprocess.call(['python', 'preprocess.py'])
if args.train:
subprocess.call(['python', 'train.py'])
if args.explain:
subprocess.call(['python', 'explain.py'])
if __name__ == "__main__":
# AGPL license agreement
try:
agpl.check_license("ripple", "LETS")
except agpl.LicenseError as e:
print(str(e))
sys.exit(1)
try:
consoleHelper.printServerStartHeader(True)
# Read config
consoleHelper.printNoNl("> Reading config file... ")
glob.conf = config.config("config.ini")
if glob.conf.default:
# We have generated a default config.ini, quit server
consoleHelper.printWarning()
consoleHelper.printColored(
"[!] config.ini not found. A default one has been generated.",
bcolors.YELLOW)
consoleHelper.printColored(
"[!] Please edit your config.ini and run the server again.",
bcolors.YELLOW)
sys.exit()
# If we haven't generated a default config.ini, check if it's valid
if not glob.conf.checkConfig():
consoleHelper.printError()
def main() -> int:
# AGPL license agreement
try:
agpl.check_license("ripple", "LETS")
except agpl.LicenseError as e:
print(str(e))
return 1
try:
consoleHelper.printServerStartHeader(True)
# Read config
consoleHelper.printNoNl("> Reading config file... ")
glob.conf = config.config("config.ini")
if glob.conf.default:
# We have generated a default config.ini, quit server
consoleHelper.printWarning()
consoleHelper.printColored(
"[!] config.ini not found. A default one has been generated.",
bcolors.YELLOW)
consoleHelper.printColored(
"[!] Please edit your config.ini and run the server again.",
bcolors.YELLOW)
return 1
# If we haven't generated a default config.ini, check if it's valid
if not glob.conf.checkConfig():
consoleHelper.printError()
consoleHelper.printColored(
"[!] Invalid config.ini. Please configure it properly",
bcolors.RED)
consoleHelper.printColored(
"[!] Delete your config.ini to generate a default one",
bcolors.RED)
return 1
else:
consoleHelper.printDone()
# Read additional config file
consoleHelper.printNoNl("> Loading additional config file... ")
try:
if not os.path.isfile(glob.conf.config["custom"]["config"]):
consoleHelper.printWarning()
consoleHelper.printColored(
"[!] Missing config file at {}; A default one has been generated at this location."
.format(glob.conf.config["custom"]["config"]),
bcolors.YELLOW)
shutil.copy("common/default_config.json",
glob.conf.config["custom"]["config"])
with open(glob.conf.config["custom"]["config"], "r") as f:
glob.conf.extra = json.load(f)
consoleHelper.printDone()
except:
consoleHelper.printWarning()
consoleHelper.printColored(
"[!] Unable to load custom config at {}".format(
glob.conf.config["custom"]["config"]), bcolors.RED)
return 1
# Create data/oppai maps folder if needed
consoleHelper.printNoNl("> Checking folders... ")
paths = [
".data", ".data/oppai", ".data/catch_the_pp",
glob.conf.config["server"]["replayspath"],
"{}_relax".format(glob.conf.config["server"]["replayspath"]),
glob.conf.config["server"]["beatmapspath"],
glob.conf.config["server"]["screenshotspath"]
]
for i in paths:
if not os.path.exists(i):
os.makedirs(i, 0o770)
consoleHelper.printDone()
# Connect to db
try:
consoleHelper.printNoNl("> Connecting to MySQL database... ")
glob.db = dbConnector.db(glob.conf.config["db"]["host"],
glob.conf.config["db"]["username"],
glob.conf.config["db"]["password"],
glob.conf.config["db"]["database"],
int(glob.conf.config["db"]["workers"]))
consoleHelper.printNoNl(" ")
consoleHelper.printDone()
except:
# Exception while connecting to db
consoleHelper.printError()
consoleHelper.printColored(
"[!] Error while connection to database. Please check your config.ini and run the server again",
bcolors.RED)
raise
# Connect to redis
try:
consoleHelper.printNoNl("> Connecting to redis... ")
glob.redis = redis.Redis(glob.conf.config["redis"]["host"],
glob.conf.config["redis"]["port"],
glob.conf.config["redis"]["database"],
glob.conf.config["redis"]["password"])
glob.redis.ping()
consoleHelper.printNoNl(" ")
consoleHelper.printDone()
except:
# Exception while connecting to db
consoleHelper.printError()
consoleHelper.printColored(
"[!] Error while connection to redis. Please check your config.ini and run the server again",
bcolors.RED)
raise
# Empty redis cache
#TODO: do we need this?
try:
glob.redis.eval(
"return redis.call('del', unpack(redis.call('keys', ARGV[1])))",
0, "lets:*")
except redis.exceptions.ResponseError:
# Script returns error if there are no keys starting with peppy:*
pass
# Save lets version in redis
glob.redis.set("lets:version", glob.VERSION)
# Create threads pool
try:
consoleHelper.printNoNl("> Creating threads pool... ")
glob.pool = ThreadPool(int(glob.conf.config["server"]["threads"]))
consoleHelper.printDone()
except:
consoleHelper.printError()
consoleHelper.printColored(
"[!] Error while creating threads pool. Please check your config.ini and run the server again",
bcolors.RED)
# Load achievements
consoleHelper.printNoNl("> Loading achievements... ")
try:
achievements = glob.db.fetchAll("SELECT * FROM achievements")
for achievement in achievements:
condition = eval(
f"lambda score, mode_vn, stats: {achievement.pop('cond')}")
glob.achievements.append(
Achievement(_id=achievement['id'],
file=achievement['icon'],
name=achievement['name'],
desc=achievement['description'],
cond=condition))
except Exception as e:
consoleHelper.printError()
consoleHelper.printColored(
"[!] Error while loading achievements! ({})".format(
traceback.format_exc()),
bcolors.RED,
)
return 1
consoleHelper.printDone()
# Set achievements version
glob.redis.set("lets:achievements_version", glob.ACHIEVEMENTS_VERSION)
consoleHelper.printColored(
"Achievements version is {}".format(glob.ACHIEVEMENTS_VERSION),
bcolors.YELLOW)
# Print disallowed mods into console (Used to also assign it into variable but has been moved elsewhere)
unranked_mods = [
key for key, value in glob.conf.extra["common"]
["rankable-mods"].items() if not value
]
consoleHelper.printColored(
"Unranked mods: {}".format(", ".join(unranked_mods)),
bcolors.YELLOW)
# Print allowed beatmap rank statuses
allowed_beatmap_rank = [
key for key, value in glob.conf.extra["lets"]
["allowed-beatmap-rankstatus"].items() if value
]
consoleHelper.printColored(
"Allowed beatmap rank statuses: {}".format(
", ".join(allowed_beatmap_rank)), bcolors.YELLOW)
# Make array of bools to respective rank id's
glob.conf.extra["_allowed_beatmap_rank"] = [
getattr(rankedStatuses, key) for key in allowed_beatmap_rank
] # Store the allowed beatmap rank id's into glob
# Discord
if generalUtils.stringToBool(glob.conf.config["discord"]["enable"]):
glob.schiavo = schiavo.schiavo(
glob.conf.config["discord"]["boturl"], "**lets**")
else:
consoleHelper.printColored(
"[!] Warning! Discord logging is disabled!", bcolors.YELLOW)
# Check debug mods
glob.debug = generalUtils.stringToBool(
glob.conf.config["server"]["debug"])
if glob.debug:
consoleHelper.printColored(
"[!] Warning! Server running in debug mode!", bcolors.YELLOW)
# Server port
try:
serverPort = int(glob.conf.config["server"]["port"])
except:
consoleHelper.printColored(
"[!] Invalid server port! Please check your config.ini and run the server again",
bcolors.RED)
# Make app
glob.application = make_app()
# Set up sentry
try:
glob.sentry = generalUtils.stringToBool(
glob.conf.config["sentry"]["enable"])
if glob.sentry:
glob.application.sentry_client = AsyncSentryClient(
glob.conf.config["sentry"]["dsn"], release=glob.VERSION)
else:
consoleHelper.printColored(
"[!] Warning! Sentry logging is disabled!", bcolors.YELLOW)
except:
consoleHelper.printColored(
"[!] Error while starting Sentry client! Please check your config.ini and run the server again",
bcolors.RED)
# Set up Datadog
try:
if generalUtils.stringToBool(
glob.conf.config["datadog"]["enable"]):
glob.dog = datadogClient.datadogClient(
glob.conf.config["datadog"]["apikey"],
glob.conf.config["datadog"]["appkey"])
else:
consoleHelper.printColored(
"[!] Warning! Datadog stats tracking is disabled!",
bcolors.YELLOW)
except:
consoleHelper.printColored(
"[!] Error while starting Datadog client! Please check your config.ini and run the server again",
bcolors.RED)
# Connect to pubsub channels
pubSub.listener(glob.redis, {
"lets:beatmap_updates": beatmapUpdateHandler.handler(),
}).start()
# Prometheus port
statsPort = None
try:
if glob.conf.config["prometheus"]["port"]:
statsPort = int(glob.conf.config["prometheus"]["port"])
except:
consoleHelper.printColored(
"Invalid stats port! Please check your config.ini and run the server again",
bcolors.YELLOW)
raise
if statsPort:
consoleHelper.printColored(
"Stats exporter listening on localhost:{}".format(statsPort),
bcolors.GREEN)
prometheus_client.start_http_server(statsPort, addr="127.0.0.1")
# Server start message and console output
consoleHelper.printColored(
"> L.E.T.S. is listening for clients on {}:{}...".format(
glob.conf.config["server"]["host"], serverPort), bcolors.GREEN)
# Start Tornado
glob.application.listen(serverPort,
address=glob.conf.config["server"]["host"])
tornado.ioloop.IOLoop.instance().start()
finally:
# Perform some clean up
print("> Disposing server... ")
glob.fileBuffers.flushAll()
consoleHelper.printColored("Goodbye!", bcolors.GREEN)
return 0
