def massRecalc(scores, workersNum=0):
"""
Recalc pp for scores in scores dictionary.
scores -- dictionary returned from query. must contain id key with score id
workersNum -- number of workers. If 0, will spawn 1 worker every 200 scores up to MAX_WORKERS
"""
# Get total scores number
totalScores = len(scores)
# Calculate number of workers if needed
if workersNum == 0:
workersNum = min(math.ceil(totalScores / 200), MAX_WORKERS)
# Start from the first score
end = 0
# Create workers list
workers = []
# Spawn necessary workers
for i in range(0, workersNum):
# Set this worker's scores range
start = end
end = start + math.floor(len(scores) / workersNum)
consoleHelper.printColored(
"> Spawning worker {} ({}:{})".format(i, start, end),
bcolors.PINK)
# Append a worker object to workers list, passing scores to recalc
workers.append(worker(i, scores[start:end]))
# Create this worker's thread and start it
t = threading.Thread(target=workers[i].doWork)
t.start()
# Infinite output loop
with progressbar.ProgressBar(widgets=[
"\033[92m",
"Progress:",
progressbar.FormatLabel(" %(value)s/%(max)s "),
progressbar.Bar(marker='#', left='[', right=']', fill='.'),
"\033[93m ",
progressbar.Percentage(),
" (",
progressbar.ETA(),
") ",
"\033[0m",
],
max_value=totalScores,
redirect_stdout=True) as bar:
while True:
# Variables needed to calculate percentage
totalPerc = 0
scoresDone = 0
workersDone = 0
# Loop through all workers
for i in range(0, workersNum):
# Get percentage, calculated scores number and done status
totalPerc += workers[i].perc
scoresDone += workers[i].current
if workers[i].done:
workersDone += 1
# Output global information
#consoleHelper.printColored("> Progress {perc:.2f}% ({done}/{total}) [{donew}/{workers}]".format(perc=totalPerc/workersNum, done=scoresDone, total=totalScores, donew=workersDone, workers=workersNum), bcolors.YELLOW)
bar.update(scoresDone)
# Exit from the loop if every worker has finished its work
if workersDone == workersNum:
break
# Repeat after 0.1 seconds
time.sleep(0.1)
print(str(e))
sys.exit(1)
try:
# Server start
consoleHelper.printServerStartHeader(True)
# Read config.ini
consoleHelper.printNoNl("> Loading config file... ")
glob.conf = configHelper.config("config.ini")
if glob.conf.default:
# We have generated a default config.ini, quit server
consoleHelper.printWarning()
consoleHelper.printColored(
"[!] config.ini not found. A default one has been generated.",
bcolors.YELLOW)
consoleHelper.printColored(
"[!] Please edit your config.ini and run the server again.",
bcolors.YELLOW)
sys.exit()
# If we haven't generated a default config.ini, check if it's valid
if not glob.conf.checkConfig():
consoleHelper.printError()
consoleHelper.printColored(
"[!] Invalid config.ini. Please configure it properly",
bcolors.RED)
consoleHelper.printColored(
"[!] Delete your config.ini to generate a default one",
bcolors.RED)
workers = int(args.workers)
# Connect to MySQL
try:
consoleHelper.printNoNl("> Connecting to MySQL db")
glob.db = dbConnector.db(glob.conf.config["db"]["host"],
glob.conf.config["db"]["username"],
glob.conf.config["db"]["password"],
glob.conf.config["db"]["database"],
max(workers, MAX_WORKERS))
consoleHelper.printNoNl(" ")
consoleHelper.printDone()
except:
consoleHelper.printError()
consoleHelper.printColored(
"[!] Error while connection to database. Please check your config.ini and run the server again",
bcolors.RED)
raise
# Set verbose
glob.debug = args.verbose
# Operations
if args.zero:
# 0pp recalc
print("> Recalculating pp for zero-pp scores")
scores = glob.db.fetchAll(
"SELECT * FROM scores LEFT JOIN beatmaps ON scores.beatmap_md5 = beatmaps.beatmap_md5 WHERE scores.completed = 3 AND scores.pp = 0 ORDER BY scores.id DESC;"
)
massRecalc(scores, workers)
elif args.cmyui:
def main() -> int:
# AGPL license agreement
try:
agpl.check_license("ripple", "LETS")
except agpl.LicenseError as e:
print(str(e))
return 1
try:
consoleHelper.printServerStartHeader(True)
# Read config
consoleHelper.printNoNl("> Reading config file... ")
glob.conf = config.config("config.ini")
if glob.conf.default:
# We have generated a default config.ini, quit server
consoleHelper.printWarning()
consoleHelper.printColored(
"[!] config.ini not found. A default one has been generated.",
bcolors.YELLOW)
consoleHelper.printColored(
"[!] Please edit your config.ini and run the server again.",
bcolors.YELLOW)
return 1
# If we haven't generated a default config.ini, check if it's valid
if not glob.conf.checkConfig():
consoleHelper.printError()
consoleHelper.printColored(
"[!] Invalid config.ini. Please configure it properly",
bcolors.RED)
consoleHelper.printColored(
"[!] Delete your config.ini to generate a default one",
bcolors.RED)
return 1
else:
consoleHelper.printDone()
# Read additional config file
consoleHelper.printNoNl("> Loading additional config file... ")
try:
if not os.path.isfile(glob.conf.config["custom"]["config"]):
consoleHelper.printWarning()
consoleHelper.printColored(
"[!] Missing config file at {}; A default one has been generated at this location."
.format(glob.conf.config["custom"]["config"]),
bcolors.YELLOW)
shutil.copy("common/default_config.json",
glob.conf.config["custom"]["config"])
with open(glob.conf.config["custom"]["config"], "r") as f:
glob.conf.extra = json.load(f)
consoleHelper.printDone()
except:
consoleHelper.printWarning()
consoleHelper.printColored(
"[!] Unable to load custom config at {}".format(
glob.conf.config["custom"]["config"]), bcolors.RED)
return 1
# Create data/oppai maps folder if needed
consoleHelper.printNoNl("> Checking folders... ")
paths = [
".data", ".data/oppai", ".data/catch_the_pp",
glob.conf.config["server"]["replayspath"],
"{}_relax".format(glob.conf.config["server"]["replayspath"]),
glob.conf.config["server"]["beatmapspath"],
glob.conf.config["server"]["screenshotspath"]
]
for i in paths:
if not os.path.exists(i):
os.makedirs(i, 0o770)
consoleHelper.printDone()
# Connect to db
try:
consoleHelper.printNoNl("> Connecting to MySQL database... ")
glob.db = dbConnector.db(glob.conf.config["db"]["host"],
glob.conf.config["db"]["username"],
glob.conf.config["db"]["password"],
glob.conf.config["db"]["database"],
int(glob.conf.config["db"]["workers"]))
consoleHelper.printNoNl(" ")
consoleHelper.printDone()
except:
# Exception while connecting to db
consoleHelper.printError()
consoleHelper.printColored(
"[!] Error while connection to database. Please check your config.ini and run the server again",
bcolors.RED)
raise
# Connect to redis
try:
consoleHelper.printNoNl("> Connecting to redis... ")
glob.redis = redis.Redis(glob.conf.config["redis"]["host"],
glob.conf.config["redis"]["port"],
glob.conf.config["redis"]["database"],
glob.conf.config["redis"]["password"])
glob.redis.ping()
consoleHelper.printNoNl(" ")
consoleHelper.printDone()
except:
# Exception while connecting to db
consoleHelper.printError()
consoleHelper.printColored(
"[!] Error while connection to redis. Please check your config.ini and run the server again",
bcolors.RED)
raise
# Empty redis cache
#TODO: do we need this?
try:
glob.redis.eval(
"return redis.call('del', unpack(redis.call('keys', ARGV[1])))",
0, "lets:*")
except redis.exceptions.ResponseError:
# Script returns error if there are no keys starting with peppy:*
pass
# Save lets version in redis
glob.redis.set("lets:version", glob.VERSION)
# Create threads pool
try:
consoleHelper.printNoNl("> Creating threads pool... ")
glob.pool = ThreadPool(int(glob.conf.config["server"]["threads"]))
consoleHelper.printDone()
except:
consoleHelper.printError()
consoleHelper.printColored(
"[!] Error while creating threads pool. Please check your config.ini and run the server again",
bcolors.RED)
# Load achievements
consoleHelper.printNoNl("> Loading achievements... ")
try:
achievements = glob.db.fetchAll("SELECT * FROM achievements")
for achievement in achievements:
condition = eval(
f"lambda score, mode_vn, stats: {achievement.pop('cond')}")
glob.achievements.append(
Achievement(_id=achievement['id'],
file=achievement['icon'],
name=achievement['name'],
desc=achievement['description'],
cond=condition))
except Exception as e:
consoleHelper.printError()
consoleHelper.printColored(
"[!] Error while loading achievements! ({})".format(
traceback.format_exc()),
bcolors.RED,
)
return 1
consoleHelper.printDone()
# Set achievements version
glob.redis.set("lets:achievements_version", glob.ACHIEVEMENTS_VERSION)
consoleHelper.printColored(
"Achievements version is {}".format(glob.ACHIEVEMENTS_VERSION),
bcolors.YELLOW)
# Print disallowed mods into console (Used to also assign it into variable but has been moved elsewhere)
unranked_mods = [
key for key, value in glob.conf.extra["common"]
["rankable-mods"].items() if not value
]
consoleHelper.printColored(
"Unranked mods: {}".format(", ".join(unranked_mods)),
bcolors.YELLOW)
# Print allowed beatmap rank statuses
allowed_beatmap_rank = [
key for key, value in glob.conf.extra["lets"]
["allowed-beatmap-rankstatus"].items() if value
]
consoleHelper.printColored(
"Allowed beatmap rank statuses: {}".format(
", ".join(allowed_beatmap_rank)), bcolors.YELLOW)
# Make array of bools to respective rank id's
glob.conf.extra["_allowed_beatmap_rank"] = [
getattr(rankedStatuses, key) for key in allowed_beatmap_rank
] # Store the allowed beatmap rank id's into glob
# Discord
if generalUtils.stringToBool(glob.conf.config["discord"]["enable"]):
glob.schiavo = schiavo.schiavo(
glob.conf.config["discord"]["boturl"], "**lets**")
else:
consoleHelper.printColored(
"[!] Warning! Discord logging is disabled!", bcolors.YELLOW)
# Check debug mods
glob.debug = generalUtils.stringToBool(
glob.conf.config["server"]["debug"])
if glob.debug:
consoleHelper.printColored(
"[!] Warning! Server running in debug mode!", bcolors.YELLOW)
# Server port
try:
serverPort = int(glob.conf.config["server"]["port"])
except:
consoleHelper.printColored(
"[!] Invalid server port! Please check your config.ini and run the server again",
bcolors.RED)
# Make app
glob.application = make_app()
# Set up sentry
try:
glob.sentry = generalUtils.stringToBool(
glob.conf.config["sentry"]["enable"])
if glob.sentry:
glob.application.sentry_client = AsyncSentryClient(
glob.conf.config["sentry"]["dsn"], release=glob.VERSION)
else:
consoleHelper.printColored(
"[!] Warning! Sentry logging is disabled!", bcolors.YELLOW)
except:
consoleHelper.printColored(
"[!] Error while starting Sentry client! Please check your config.ini and run the server again",
bcolors.RED)
# Set up Datadog
try:
if generalUtils.stringToBool(
glob.conf.config["datadog"]["enable"]):
glob.dog = datadogClient.datadogClient(
glob.conf.config["datadog"]["apikey"],
glob.conf.config["datadog"]["appkey"])
else:
consoleHelper.printColored(
"[!] Warning! Datadog stats tracking is disabled!",
bcolors.YELLOW)
except:
consoleHelper.printColored(
"[!] Error while starting Datadog client! Please check your config.ini and run the server again",
bcolors.RED)
# Connect to pubsub channels
pubSub.listener(glob.redis, {
"lets:beatmap_updates": beatmapUpdateHandler.handler(),
}).start()
# Prometheus port
statsPort = None
try:
if glob.conf.config["prometheus"]["port"]:
statsPort = int(glob.conf.config["prometheus"]["port"])
except:
consoleHelper.printColored(
"Invalid stats port! Please check your config.ini and run the server again",
bcolors.YELLOW)
raise
if statsPort:
consoleHelper.printColored(
"Stats exporter listening on localhost:{}".format(statsPort),
bcolors.GREEN)
prometheus_client.start_http_server(statsPort, addr="127.0.0.1")
# Server start message and console output
consoleHelper.printColored(
"> L.E.T.S. is listening for clients on {}:{}...".format(
glob.conf.config["server"]["host"], serverPort), bcolors.GREEN)
# Start Tornado
glob.application.listen(serverPort,
address=glob.conf.config["server"]["host"])
tornado.ioloop.IOLoop.instance().start()
finally:
# Perform some clean up
print("> Disposing server... ")
glob.fileBuffers.flushAll()
consoleHelper.printColored("Goodbye!", bcolors.GREEN)
return 0
sys.exit(1)
try:
# Server start
consoleHelper.printServerStartHeader(True)
# Read config.ini
consoleHelper.printNoNl(
"> Loading config file [/] ")
glob.conf = configHelper.config("config.ini")
if glob.conf.default:
# We have generated a default config.ini, quit server
consoleHelper.printWarning()
consoleHelper.printColored(
"[!] config.ini not found. A default one has been generated.",
bcolors.YELLOW)
consoleHelper.printColored(
"[!] Please edit your config.ini and run the server again.",
bcolors.YELLOW)
sys.exit()
# If we haven't generated a default config.ini, check if it's valid
if not glob.conf.checkConfig():
consoleHelper.printError()
consoleHelper.printColored(
"[!] Invalid config.ini. Please configure it properly",
bcolors.RED)
consoleHelper.printColored(
"[!] Delete your config.ini to generate a default one",
bcolors.RED)
except agpl.LicenseError as e:
print(str(e))
sys.exit(1)
try:
consoleHelper.printServerStartHeader(True)
# Read config
consoleHelper.printNoNl("> Reading config file... ")
glob.conf = config.config("config.ini")
if glob.conf.default:
# We have generated a default config.ini, quit server
consoleHelper.printWarning()
consoleHelper.printColored(
"[!] config.ini not found. A default one has been generated.",
bcolors.YELLOW)
consoleHelper.printColored(
"[!] Please edit your config.ini and run the server again.",
bcolors.YELLOW)
sys.exit()
# If we haven't generated a default config.ini, check if it's valid
if not glob.conf.checkConfig():
consoleHelper.printError()
consoleHelper.printColored(
"[!] Invalid config.ini. Please configure it properly",
bcolors.RED)
consoleHelper.printColored(
"[!] Delete your config.ini to generate a default one",
bcolors.RED)
default_handler_class=defaultHandler.handler)
if __name__ == "__main__":
try:
consoleHelper.printServerStartHeader(True)
# Read config
consoleHelper.printNoNl("> Reading config file... ")
glob.conf = config.config("config.ini")
if glob.conf.default:
# We have generated a default config.ini, quit server
consoleHelper.printWarning()
consoleHelper.printColored(
"[!] config.ini not found. A default one has been generated.",
bcolors.YELLOW)
consoleHelper.printColored(
"[!] Please edit your config.ini and run the server again.",
bcolors.YELLOW)
sys.exit()
# If we haven't generated a default config.ini, check if it's valid
if not glob.conf.checkConfig():
consoleHelper.printError()
consoleHelper.printColored(
"[!] Invalid config.ini. Please configure it properly",
bcolors.RED)
consoleHelper.printColored(
"[!] Delete your config.ini to generate a default one",
bcolors.RED)
def getPP(self, tillerino=False, stars=False):
"""
Calculate total pp value with oppai and return it
return -- total pp
"""
# Set variables
self.pp = 0
try:
# Build .osu map file path
mapFile = "{path}/maps/{map}".format(path=self.OPPAI_FOLDER,
map=self.map)
try:
# Check if we have to download the .osu file
download = False
if not os.path.isfile(mapFile):
# .osu file doesn't exist. We must download it
if glob.debug:
consoleHelper.printColored(
"[!] {} doesn't exist".format(mapFile),
bcolors.YELLOW)
download = True
else:
# File exists, check md5
if generalUtils.fileMd5(mapFile) != self.beatmap.fileMD5:
# MD5 don't match, redownload .osu file
if glob.debug:
consoleHelper.printColored(
"[!] Beatmaps md5 don't match", bcolors.YELLOW)
download = True
# Download .osu file if needed
if download:
if glob.debug:
consoleHelper.printRippoppaiMessage(
"Downloading {} from osu! servers...".format(
self.beatmap.beatmapID))
# Get .osu file from osu servers
fileContent = osuapiHelper.getOsuFileFromID(
self.beatmap.beatmapID)
# Make sure osu servers returned something
if fileContent is None:
raise exceptions.osuApiFailException(MODULE_NAME)
# Delete old .osu file if it exists
if os.path.isfile(mapFile):
os.remove(mapFile)
# Save .osu file
with open(mapFile, "wb+") as f:
f.write(fileContent.encode("latin-1"))
else:
# Map file is already in folder
if glob.debug:
consoleHelper.printRippoppaiMessage(
"Found beatmap file {}".format(mapFile))
except exceptions.osuApiFailException:
pass
# Base command
command = fixPath("{path}/oppai {mapFile}".format(
path=self.OPPAI_FOLDER, mapFile=mapFile))
# Use only mods supported by oppai.
modsFixed = self.mods & 5979
# Add params if needed
if self.acc > 0:
command += " {acc:.2f}%".format(acc=self.acc)
if self.mods > 0:
command += " +{mods}".format(
mods=scoreUtils.readableMods(modsFixed))
if self.combo > 0:
command += " {combo}x".format(combo=self.combo)
if self.misses > 0:
command += " {misses}xm".format(misses=self.misses)
if tillerino:
command += " tillerino"
if stars:
command += " stars"
# Debug output
if glob.debug:
consoleHelper.printRippoppaiMessage(
"Executing {}".format(command))
# oppai output
process = subprocess.run(command,
shell=True,
stdout=subprocess.PIPE)
output = process.stdout.decode("utf-8")
# Get standard or tillerino output
sep = "\n" if UNIX else "\r\n"
if output == ['']:
# This happens if mode not supported or something
self.pp = 0
self.stars = None
return self.pp
output = output.split(sep)
# get rid of pesky warnings!!!
try:
float(output[0])
except ValueError:
del output[0]
if tillerino:
# Get tillerino output (multiple lines)
if stars:
self.pp = output[:-2]
self.stars = float(output[-2])
else:
self.pp = output.split(
sep)[:
-1] # -1 because there's an empty line at the end
else:
# Get standard output (:l to remove (/r)/n at the end)
l = -1 if UNIX else -2
if stars:
self.pp = float(output[len(output) - 2][:l - 1])
else:
self.pp = float(output[len(output) - 2][:l])
# Debug output
if glob.debug:
consoleHelper.printRippoppaiMessage("Calculated pp: {}".format(
self.pp))
finally:
return self.pp
import time
if __name__ == "__main__":
try:
# Server start
consoleHelper.printServerStartHeader(True)
# Read config.ini
consoleHelper.printNoNl("> Loading config file...")
glob.conf = configHelper.config("config.ini")
if glob.conf.default:
# We have generated a default config.ini, quit server
consoleHelper.printWarning()
consoleHelper.printColored(
"[!] config.ini not found. A default one has been generated.",
bcolors.YELLOW)
consoleHelper.printColored(
"[!] Please edit your config.ini and run the server again.",
bcolors.YELLOW)
sys.exit()
# If we haven't generated a default config.ini, check if it's valid
if not glob.conf.checkConfig():
consoleHelper.printError()
consoleHelper.printColored(
"[!] Invalid config.ini. Please configure it properly.",
bcolors.RED)
consoleHelper.printColored(
"[!] Delete your config.ini to generate a default one.",
bcolors.RED)
def calculatePP(self):
"""
Calculate total pp value with oppai and return it
return -- total pp
"""
# Set variables
self.pp = None
try:
# Build .osu map file path
mapFile = "{path}/maps/{map}".format(path=self.OPPAI_FOLDER, map=self.map)
log.debug("oppai ~> Map file: {}".format(mapFile))
try:
# Check if we have to download the .osu file
download = False
if not os.path.isfile(mapFile):
# .osu file doesn't exist. We must download it
if glob.debug:
consoleHelper.printColored("[!] {} doesn't exist".format(mapFile), bcolors.YELLOW)
download = True
else:
# File exists, check md5
if generalUtils.fileMd5(mapFile) != self.beatmap.fileMD5:
# MD5 don't match, redownload .osu file
if glob.debug:
consoleHelper.printColored("[!] Beatmaps md5 don't match", bcolors.YELLOW)
download = True
# Download .osu file if needed
if download:
log.debug("oppai ~> Downloading {} osu file".format(self.beatmap.beatmapID))
# Get .osu file from osu servers
fileContent = osuapiHelper.getOsuFileFromID(self.beatmap.beatmapID)
# Make sure osu servers returned something
if fileContent is None:
raise exceptions.osuApiFailException(MODULE_NAME)
# Delete old .osu file if it exists
if os.path.isfile(mapFile):
os.remove(mapFile)
# Save .osu file
with open(mapFile, "wb+") as f:
f.write(fileContent.encode("utf-8"))
else:
# Map file is already in folder
log.debug("oppai ~> Beatmap found in cache!")
except exceptions.osuApiFailException:
log.error("oppai ~> osu!api error!")
pass
# Parse beatmap
log.debug("oppai ~> About to parse beatmap")
pyoppai.parse(
mapFile,
self._oppai_beatmap,
self._oppai_buffer,
self.BUFSIZE,
False,
self.OPPAI_FOLDER # /oppai_cache
)
self.checkOppaiErrors()
log.debug("oppai ~> Beatmap parsed with no errors")
# Create diffcalc context and calculate difficulty
log.debug("oppai ~> About to calculate difficulty")
# Use only mods supported by oppai
modsFixed = self.mods & 5979
if modsFixed > 0:
pyoppai.apply_mods(self._oppai_beatmap, modsFixed)
self._oppai_diffcalc_ctx = pyoppai.new_d_calc_ctx(self._oppai_ctx)
diff_stars, diff_aim, diff_speed, _, _, _, _ = pyoppai.d_calc(self._oppai_diffcalc_ctx, self._oppai_beatmap)
self.checkOppaiErrors()
log.debug("oppai ~> Difficulty calculated with no errors. {}*, {} aim, {} speed".format(diff_stars, diff_aim, diff_speed))
# Calculate pp
log.debug("oppai ~> About to calculate PP")
if not self.tillerino:
_, total_pp, aim_pp, speed_pp, acc_pp = pyoppai.pp_calc_acc(self._oppai_ctx, diff_aim, diff_speed, self._oppai_beatmap,
self.acc if self.acc > 0 else 100,
modsFixed,
self.combo if self.combo > 0 else 0xFFFF,
self.misses)
self.checkOppaiErrors()
log.debug("oppai ~> PP Calculated with no errors. {}pp, {} aim pp, {} speed pp, {} acc pp".format(
total_pp, aim_pp, speed_pp, acc_pp
))
self.pp = total_pp
else:
pp_list = []
for acc in [100, 99, 98, 95]:
log.debug("oppai ~> Calculating PP with acc {}%".format(acc))
_, total_pp, aim_pp, speed_pp, acc_pp = pyoppai.pp_calc_acc(self._oppai_ctx, diff_aim, diff_speed,
self._oppai_beatmap, acc, modsFixed)
self.checkOppaiErrors()
pp_list.append(total_pp)
log.debug("oppai ~> PP Calculated with no errors. {}pp, {} aim pp, {} speed pp, {} acc pp".format(
total_pp, aim_pp, speed_pp, acc_pp
))
self.pp = pp_list
self.stars = diff_stars
log.debug("oppai ~> Calculated PP: {}".format(self.pp))
except OppaiError:
log.error("oppai ~> pyoppai error!")
self.pp = 0
except Exception as e:
log.error("oppai ~> Unhandled exception: {}".format(str(e)))
raise e
finally:
log.debug("oppai ~> Shutting down and returning {}pp".format(self.pp))
return self.pp