def fix_missing_winner(replay):
logger.info("fix_missing_winner(%s)", replay)
SldbMatchSkillsCache.objects.filter(gameID=replay.gameID).delete()
try:
match_skills = get_sldb_match_skills([replay.gameID])
if match_skills:
match_skills = match_skills[0]
except SLDBConnectionError as exc:
logger.error("get_sldb_match_skills(%s): %s", [replay.gameID], exc)
return
if match_skills and match_skills["status"] == 0:
match_skills_by_pa = dict()
for player in match_skills["players"]:
match_skills_by_pa[player["account"]] = [s[0] for s in player["skills"][:2]]
else:
logger.error("No TS data for match from SLDB available, no fix possible.")
return
for at in replay.allyteam_set.all():
pas = [team.teamleader.account for team in at.team_set.all()]
at_sum_old = sum(match_skills_by_pa[pa][0] for pa in pas)
at_sum_new = sum(match_skills_by_pa[pa][1] for pa in pas)
logger.info("Allyteam %s has rating change %r -> %r.", at, at_sum_old, at_sum_new)
if at_sum_new > at_sum_old:
logger.info("Allyteam %s has won.", at)
at.winner = True
at.save()
else:
logger.info("Allyteam %s has lost.", at)
at.winner = False
at.save()
def rate_matches(replays):
"""
Mass rating. Fetches for efficiency 32 matches at once from SLDB.
"""
logger.info("Rating %d matches", replays.count())
current32 = list()
counter = 0
for replay in replays:
current32.append(replay)
if counter < 31:
counter += 1
else:
counter = 0
try:
_ = get_sldb_match_skills([r.gameID for r in current32])
except:
logger.exception("FIXME: to broad exception handling.")
pass
for match in current32:
try:
rate_match(match)
except Exception, e:
logger.exception(e)
logger.info("done 32")
current32 = list()
def fix_missing_winner(replay):
logger.info("fix_missing_winner(%s)", replay)
SldbMatchSkillsCache.objects.filter(gameID=replay.gameID).delete()
try:
match_skills = get_sldb_match_skills([replay.gameID])
if match_skills:
match_skills = match_skills[0]
except SLDBConnectionError as exc:
logger.error("get_sldb_match_skills(%s): %s", [replay.gameID], exc)
return
if match_skills and match_skills["status"] == 0:
match_skills_by_pa = dict()
for player in match_skills["players"]:
match_skills_by_pa[player["account"]] = [
s[0] for s in player["skills"][:2]
]
else:
logger.error(
"No TS data for match from SLDB available, no fix possible.")
return
for at in replay.allyteam_set.all():
pas = [team.teamleader.account for team in at.team_set.all()]
at_sum_old = sum(match_skills_by_pa[pa][0] for pa in pas)
at_sum_new = sum(match_skills_by_pa[pa][1] for pa in pas)
logger.info("Allyteam %s has rating change %r -> %r.", at, at_sum_old,
at_sum_new)
if at_sum_new > at_sum_old:
logger.info("Allyteam %s has won.", at)
at.winner = True
at.save()
else:
logger.info("Allyteam %s has lost.", at)
at.winner = False
at.save()
def replay(request, gameID):
c = all_page_infos(request)
if not gameid_re.findall(gameID):
raise Http404("Malformed gameID: %r." % gameID)
try:
replay = Replay.objects.prefetch_related().get(gameID=gameID)
c["replay"] = replay
c["comment_obj"] = replay
except ObjectDoesNotExist:
raise Http404("No replay with gameID '" + strip_tags(gameID) + "' found.")
if not replay.published:
return render(request, 'replay_unpublished.html', c)
game = replay.game_release.game
match_type = replay.match_type_short
try:
match_skills = get_sldb_match_skills([replay.gameID])
if match_skills:
match_skills = match_skills[0]
except SLDBConnectionError as exc:
logger.error("get_sldb_match_skills(%s): %s", [replay.gameID], exc)
match_skills = {"status": 3}
# ignore, we'll just use the old values from the DB in the view
else:
if match_skills and match_skills["status"] == 0:
# update skill data in DB
logger.debug("got match data for %s from sldb", replay)
for player in match_skills["players"]:
pa = player["account"]
pa_skill = pa.get_rating(game, match_type)
mu, si = player["skills"][1]
if pa_skill.trueskill_mu != mu or pa_skill.trueskill_sigma != si:
try:
playername = Player.objects.get(account=pa, replay=replay, spectator=False).name
except ObjectDoesNotExist:
playername = "??"
pa_skill.trueskill_mu = mu
pa_skill.trueskill_sigma = si
if pa_skill.playername == "" or pa_skill.playername == "??":
pa_skill.playername = playername
pa_skill.save()
defaults = dict(
match_date=replay.unixTime,
playername=playername,
trueskill_mu=mu,
trueskill_sigma=si
)
RatingHistory.objects.update_or_create(match=replay,
game=game,
match_type=match_type,
playeraccount=pa,
defaults=defaults)
if pa.sldb_privacy_mode != player["privacyMode"]:
pa.sldb_privacy_mode = player["privacyMode"]
pa.save()
if not replay.rated:
replay.rated = True
replay.save()
else:
# ignore, we'll just use the old values from the DB in the view
logger.debug("no match data from SLDB")
pass
# fill cache prefetching all entries from DB in one call
all_players = Player.objects.filter(replay=replay)
allyteams = Allyteam.objects.filter(replay=replay)
if not allyteams.filter(winner=True).exists() or (
replay.upload_date.year >= 2016 and allyteams.filter(winner=True, num=0).exists()):
# workaround for issue #89: guess winner from ratings
fix_missing_winner(replay)
c["allyteams"] = []
match_rating_history = RatingHistory.objects.filter(match=replay, match_type=match_type)
[entry for entry in match_rating_history] # prefetch all ratings of this match by hitting the DB only once
for at in allyteams:
playeraccounts = PlayerAccount.objects.filter(player__team__allyteam=at).order_by("player__team__num")
teams = Team.objects.filter(allyteam=at)
players = all_players.filter(account__in=playeraccounts)
players_w_rating = list()
old_rating = 0
new_rating = 0
lobby_rank_sum = 0
if replay.rated == False or replay.notcomplete or not players.exists() or not replay.game_release.game.sldb_name or all_players.filter(
account__accountid=0).exists():
# notcomplete, no SLDB rating or bot present -> no rating
players_w_rating = [(player, None, None) for player in players]
else:
# TrueSkill ratings
for pa in playeraccounts:
if match_skills and match_skills["status"] == 0:
# use SLDB-provided values
def _get_players_skills(pa):
for pl in match_skills["players"]:
if pl["account"] == pa: return pl
pl = _get_players_skills(pa)
pl_new = pl["skills"][1][0]
pl_old = pl["skills"][0][0]
else:
# use old method of DB lookups for current and previous matchs DB entries
try:
pl_new = match_rating_history.get(playeraccount=pa).trueskill_mu
except ObjectDoesNotExist:
# no rating on this replay
pl_new = None
try:
# find previous TS value
pl_old = RatingHistory.objects.filter(playeraccount=pa, game=game, match_type=match_type,
match__unixTime__lt=replay.unixTime).order_by(
"-match__unixTime")[0].trueskill_mu
except IndexError:
pl_old = 25 # default value for new players
# privatize?
if playeraccounts.count() > 2 or pa.sldb_privacy_mode == 0:
new_rating += pl_new if pl_new else 0
old_rating += pl_old if pl_old else 0
else:
new_rating += privatize_skill(pl_new) if pl_new else 0
old_rating += privatize_skill(pl_old) if pl_old else 0
if pa.sldb_privacy_mode != 0 and (
not request.user.is_authenticated() or pa.accountid != request.user.userprofile.accountid):
if pl_new:
pl_new = privatize_skill(pl_new)
if pl_old:
pl_old = privatize_skill(pl_old)
players_w_rating.append((all_players.get(account=pa, spectator=False), pl_old, pl_new))
if teams:
lobby_rank_sum = reduce(lambda x, y: x + y, [pl.rank for pl in all_players.filter(team__allyteam=at)], 0)
c["allyteams"].append((at, players_w_rating, old_rating, new_rating, lobby_rank_sum))
c["has_bot"] = replay.tags.filter(name="Bot").exists()
c["specs"] = all_players.filter(replay=replay, spectator=True).order_by("id")
c["upload_broken"] = UploadTmp.objects.filter(replay=replay).exists()
c["mapoptions"] = MapOption.objects.filter(replay=replay).order_by("name")
c["modoptions"] = ModOption.objects.filter(replay=replay).order_by("name")
c["replay_details"] = True
c["was_stopped"] = not allyteams.filter(winner=True).exists()
if c["was_stopped"]:
logger.info("was_stopped=True: allyteams=%r replay=%r", allyteams, replay)
c["is_draw"] = allyteams.filter(winner=True).count() > 1
c["pagedescription"] = "%s %s %s match on %s (%s)" % (
replay.num_players, replay.match_type, replay.game_release.game.name, replay.map_info.name, replay.unixTime)
c["replay_owners"] = get_owner_list(replay.uploader)
c["extra_media"] = ExtraReplayMedia.objects.filter(replay=replay)
c["known_video_formats"] = ["video/webm", "video/mp4", "video/ogg", "video/x-flv", "application/ogg"]
c["has_video"] = c["extra_media"].filter(media_magic_mime__in=c["known_video_formats"]).exists()
c["metadata"] = list()
if replay.map_info.width > 128:
# api.springfiles.com returnd pixel size
map_px_x = replay.map_info.width / 512
map_px_y = replay.map_info.height / 512
else:
# api.springfiles.com returnd Spring Map Size
map_px_x = replay.map_info.width
map_px_y = replay.map_info.height
try:
c["metadata"].append(("Size", "{} x {}".format(map_px_x, map_px_y)))
try:
c["metadata"].append(("Wind", "{} - {}".format(replay.map_info.metadata["metadata"]["MinWind"],
replay.map_info.metadata["metadata"]["MaxWind"])))
except KeyError:
pass
try:
c["metadata"].append(("Tidal", str(replay.map_info.metadata["metadata"]["TidalStrength"])))
except KeyError:
pass
for k, v in replay.map_info.metadata["metadata"].items():
if type(v) == str and not v.strip():
continue
elif type(v) == list and not v:
continue
elif k.strip() in ["", "Width", "TidalStrength", "MapFileName", "MapMinHeight", "Type", "MapMaxHeight",
"Resources", "Height", "MinWind", "MaxWind", "StartPos"]:
# either already added above, or ignore uninteresting data
continue
else:
c["metadata"].append((k.strip(), v))
try:
if replay.map_info.metadata["version"]:
c["metadata"].append(("Version", replay.map_info.metadata["version"]))
except KeyError:
pass
except Exception as exc:
c["metadata"].append(("Error", "Problem with metadata. Please report to Dansan."))
logger.error("FIXME: to broad exception handling.")
logger.error("Problem with metadata (replay.id '%d'), replay.map_info.metadata: %s", replay.id,
replay.map_info.metadata)
logger.exception("Exception: %s", exc)
c["xtaward_heroes"] = XTAwards.objects.filter(replay=replay, isAlive=1)
c["xtaward_los"] = XTAwards.objects.filter(replay=replay, isAlive=0)
page_history = request.session.get("page_history")
if page_history and isinstance(page_history, list):
# check data (session data is user input)
for page in list(page_history):
if not gameid_re.findall(page):
page_history.remove(page)
if gameID not in page_history:
if len(page_history) > 4:
page_history.remove(page)
page_history.insert(0, gameID)
else:
page_history = [gameID]
request.session["page_history"] = page_history
return render(request, 'replay.html', c)
def replay(request, gameID):
c = all_page_infos(request)
if not gameid_re.findall(gameID):
raise Http404("Malformed gameID: %r." % gameID)
try:
replay = Replay.objects.prefetch_related().get(gameID=gameID)
c["replay"] = replay
c["comment_obj"] = replay
except ObjectDoesNotExist:
raise Http404("No replay with gameID '" + strip_tags(gameID) +
"' found.")
if not replay.published:
return render(request, 'replay_unpublished.html', c)
game = replay.game_release.game
match_type = replay.match_type_short
sldb_connection_error = False
try:
match_skills = get_sldb_match_skills([replay.gameID])
if match_skills:
match_skills = match_skills[0]
except SLDBConnectionError as exc:
sldb_connection_error = True
logger.error("get_sldb_match_skills(%s): %s", [replay.gameID], exc)
match_skills = {"status": 3}
# ignore, we'll just use the old values from the DB in the view
else:
if match_skills and match_skills["status"] == 0:
# update skill data in DB
logger.debug("got match data for %s from sldb", replay)
for player in match_skills["players"]:
pa = player["account"]
pa_skill = pa.get_rating(game, match_type)
mu, si = player["skills"][1]
if pa_skill.trueskill_mu != mu or pa_skill.trueskill_sigma != si:
try:
playername = Player.objects.get(account=pa,
replay=replay,
spectator=False).name
except ObjectDoesNotExist:
playername = "??"
pa_skill.trueskill_mu = mu
pa_skill.trueskill_sigma = si
if pa_skill.playername == "" or pa_skill.playername == "??":
pa_skill.playername = playername
pa_skill.save()
defaults = dict(match_date=replay.unixTime,
playername=playername,
trueskill_mu=mu,
trueskill_sigma=si)
RatingHistory.objects.update_or_create(
match=replay,
game=game,
match_type=match_type,
playeraccount=pa,
defaults=defaults)
if pa.sldb_privacy_mode != player["privacyMode"]:
pa.sldb_privacy_mode = player["privacyMode"]
pa.save()
if not replay.rated:
replay.rated = True
replay.save()
else:
# ignore, we'll just use the old values from the DB in the view
logger.debug("no match data from SLDB")
pass
# fill cache prefetching all entries from DB in one call
all_players = Player.objects.filter(replay=replay)
allyteams = Allyteam.objects.filter(replay=replay)
if not allyteams.filter(winner=True).exists() or (
replay.upload_date.year >= 2016 and allyteams.filter(
winner=True, num=0).exists()) and not sldb_connection_error:
# workaround for issue #89: guess winner from ratings
fix_missing_winner(replay)
c["allyteams"] = []
match_rating_history = RatingHistory.objects.filter(match=replay,
match_type=match_type)
[entry for entry in match_rating_history
] # prefetch all ratings of this match by hitting the DB only once
for at in allyteams:
playeraccounts = PlayerAccount.objects.filter(
player__team__allyteam=at).order_by("player__team__num")
teams = Team.objects.filter(allyteam=at)
players = all_players.filter(account__in=playeraccounts)
players_w_rating = list()
old_rating = 0
new_rating = 0
lobby_rank_sum = 0
if replay.rated == False or replay.notcomplete or not players.exists(
) or not replay.game_release.game.sldb_name or all_players.filter(
account__accountid=0).exists():
# notcomplete, no SLDB rating or bot present -> no rating
players_w_rating = [(player, None, None) for player in players]
else:
# TrueSkill ratings
for pa in playeraccounts:
if match_skills and match_skills["status"] == 0:
# use SLDB-provided values
def _get_players_skills(pa):
for pl in match_skills["players"]:
if pl["account"] == pa: return pl
pl = _get_players_skills(pa)
pl_new = pl["skills"][1][0]
pl_old = pl["skills"][0][0]
else:
# use old method of DB lookups for current and previous matchs DB entries
try:
pl_new = match_rating_history.get(
playeraccount=pa).trueskill_mu
except ObjectDoesNotExist:
# no rating on this replay
pl_new = None
try:
# find previous TS value
pl_old = RatingHistory.objects.filter(
playeraccount=pa,
game=game,
match_type=match_type,
match__unixTime__lt=replay.unixTime).order_by(
"-match__unixTime")[0].trueskill_mu
except IndexError:
pl_old = 25 # default value for new players
# privatize?
if playeraccounts.count() > 2 or pa.sldb_privacy_mode == 0:
new_rating += pl_new if pl_new else 0
old_rating += pl_old if pl_old else 0
else:
new_rating += privatize_skill(pl_new) if pl_new else 0
old_rating += privatize_skill(pl_old) if pl_old else 0
if pa.sldb_privacy_mode != 0 and (
not request.user.is_authenticated()
or pa.accountid != request.user.userprofile.accountid):
if pl_new:
pl_new = privatize_skill(pl_new)
if pl_old:
pl_old = privatize_skill(pl_old)
players_w_rating.append(
(all_players.get(account=pa,
spectator=False), pl_old, pl_new))
if teams:
lobby_rank_sum = reduce(
lambda x, y: x + y,
[pl.rank for pl in all_players.filter(team__allyteam=at)], 0)
c["allyteams"].append(
(at, players_w_rating, old_rating, new_rating, lobby_rank_sum))
c["has_bot"] = replay.tags.filter(name="Bot").exists()
c["specs"] = all_players.filter(replay=replay,
spectator=True).order_by("id")
c["upload_broken"] = UploadTmp.objects.filter(replay=replay).exists()
c["mapoptions"] = MapOption.objects.filter(replay=replay).order_by("name")
c["modoptions"] = ModOption.objects.filter(replay=replay).order_by("name")
c["replay_details"] = True
c["was_stopped"] = not allyteams.filter(winner=True).exists()
if c["was_stopped"]:
logger.info("was_stopped=True: allyteams=%r replay=%r", allyteams,
replay)
c["is_draw"] = allyteams.filter(winner=True).count() > 1
c["pagedescription"] = "%s %s %s match on %s (%s)" % (
replay.num_players, replay.match_type, replay.game_release.game.name,
replay.map_info.name, replay.unixTime)
c["replay_owners"] = get_owner_list(replay.uploader)
c["extra_media"] = ExtraReplayMedia.objects.filter(replay=replay)
c["known_video_formats"] = [
"video/webm", "video/mp4", "video/ogg", "video/x-flv",
"application/ogg"
]
c["has_video"] = c["extra_media"].filter(
media_magic_mime__in=c["known_video_formats"]).exists()
c["metadata"] = list()
if replay.map_info.width > 128:
# api.springfiles.com returnd pixel size
map_px_x = replay.map_info.width / 512
map_px_y = replay.map_info.height / 512
else:
# api.springfiles.com returnd Spring Map Size
map_px_x = replay.map_info.width
map_px_y = replay.map_info.height
try:
c["metadata"].append(("Size", "{} x {}".format(map_px_x, map_px_y)))
try:
c["metadata"].append(("Wind", "{} - {}".format(
replay.map_info.metadata["metadata"]["MinWind"],
replay.map_info.metadata["metadata"]["MaxWind"])))
except KeyError:
pass
try:
c["metadata"].append(
("Tidal",
str(replay.map_info.metadata["metadata"]["TidalStrength"])))
except KeyError:
pass
for k, v in replay.map_info.metadata["metadata"].items():
if type(v) == str and not v.strip():
continue
elif type(v) == list and not v:
continue
elif k.strip() in [
"", "Width", "TidalStrength", "MapFileName",
"MapMinHeight", "Type", "MapMaxHeight", "Resources",
"Height", "MinWind", "MaxWind", "StartPos"
]:
# either already added above, or ignore uninteresting data
continue
else:
c["metadata"].append((k.strip(), v))
try:
if replay.map_info.metadata["version"]:
c["metadata"].append(
("Version", replay.map_info.metadata["version"]))
except KeyError:
pass
except Exception as exc:
c["metadata"].append(
("Error", "Problem with metadata. Please report to Dansan."))
logger.error("FIXME: to broad exception handling.")
logger.error(
"Problem with metadata (replay.id '%d'), replay.map_info.metadata: %s",
replay.id, replay.map_info.metadata)
logger.exception("Exception: %s", exc)
c["xtaward_heroes"] = XTAwards.objects.filter(replay=replay, isAlive=1)
c["xtaward_los"] = XTAwards.objects.filter(replay=replay, isAlive=0)
page_history = request.session.get("page_history")
if page_history and isinstance(page_history, list):
# check data (session data is user input)
for page in list(page_history):
if not gameid_re.findall(page):
page_history.remove(page)
if gameID not in page_history:
if len(page_history) > 4:
page_history.remove(page)
page_history.insert(0, gameID)
else:
page_history = [gameID]
request.session["page_history"] = page_history
return render(request, 'replay.html', c)
def rate_match(replay):
game = replay.game
try:
match_skill = get_sldb_match_skills([replay.gameID])
if match_skill:
match_skill = match_skill[0]
if match_skill["status"] != 0:
raise Exception(
"SLDB returned status=%d -> 1: invalid gameID value, 2: unknown or unrated gameID" % match_skill[
"status"])
else:
raise Exception("no SLDB data")
except SLDBConnectionError as exc:
logger.error("in get_sldb_match_skills([%r]): %s", replay.gameID, exc)
# use "skill" tag from demo data if available
logger.info("Trying to use skill tag from demofile")
players = Player.objects.filter(replay=replay, spectator=False).exclude(skill="")
if players.exists():
replay.rated = True
for player in players:
pa_rating = player.account.get_rating(game, replay.match_type_short)
demo_skill = demoskill2float(player.skill)
if pa_rating.trueskill_mu != demo_skill:
pa_rating.trueskill_mu = demo_skill
pa_rating.save()
if pa_rating.playername == "" or pa_rating.playername == "??":
pa_rating.playername = player.name
pa_rating.save()
defaults = {"trueskill_mu": pa_rating.trueskill_mu,
"trueskill_sigma": pa_rating.trueskill_sigma,
"playername": player.name,
"match_date": replay.unixTime}
RatingHistory.objects.update_or_create(playeraccount=player.account, match=replay, game=game,
match_type=replay.match_type_short, defaults=defaults)
else:
logger.info(".. no skill tags found.")
replay.rated = False
replay.save()
return
try:
match_type = sldb_gametype2matchtype[match_skill["gameType"]]
except:
logger.exception("FIXME: to broad exception handling.")
match_type = replay.match_type_short
for player in match_skill["players"]:
pa = player["account"]
try:
playername = Player.objects.get(account=pa, replay=replay).name
except ObjectDoesNotExist:
playername = ""
if pa.sldb_privacy_mode != player["privacyMode"]:
pa.sldb_privacy_mode = player["privacyMode"]
pa.save()
muAfter, sigmaAfter = player["skills"][1]
globalMuAfter, globalSigmaAfter = player["skills"][3]
pa_rating = pa.get_rating(game, match_type)
if pa_rating.trueskill_mu != muAfter or pa_rating.trueskill_sigma != sigmaAfter:
pa_rating.trueskill_mu = muAfter
pa_rating.trueskill_sigma = sigmaAfter
pa_rating.save()
if (pa_rating.playername == "" or pa_rating.playername == "??") and playername:
pa_rating.playername = playername
pa_rating.save()
defaults = {"trueskill_mu": muAfter,
"trueskill_sigma": sigmaAfter,
"playername": playername,
"match_date": replay.unixTime}
RatingHistory.objects.update_or_create(playeraccount=pa, match=replay, game=game, match_type=match_type, defaults=defaults)
pa_rating = pa.get_rating(game, "L") # Global
if pa_rating.trueskill_mu != globalMuAfter or pa_rating.trueskill_sigma != globalSigmaAfter:
pa_rating.trueskill_mu = globalMuAfter
pa_rating.trueskill_sigma = globalSigmaAfter
pa_rating.save()
defaults = {"trueskill_mu": globalMuAfter,
"trueskill_sigma": globalSigmaAfter,
"playername": playername,
"match_date": replay.unixTime}
RatingHistory.objects.update_or_create(playeraccount=pa, match=replay, game=game, match_type="L", defaults=defaults)
if not replay.rated:
replay.rated = True
replay.save()
logger.info("Replay(%d) %s rated with values from SLDB.", replay.id, replay.gameID)