def decide_on_players(
bot_ids: Iterable[BotID], rank_sys: RankingSystem,
ticket_sys: TicketSystem) -> Tuple[List[BotID], List[BotID]]:
"""
Find two balanced teams. The TicketSystem and the RankingSystem to find
a fair match up between some bots that haven't played for a while.
"""
limit = 200
tries_left = limit
while tries_left > 0:
tries_left -= 1
# Pick some bots that haven't played for a while
picked = ticket_sys.pick_bots(bot_ids)
shuffle(picked)
ratings = [rank_sys.get(bot) for bot in picked]
blue = tuple(ratings[0:3])
orange = tuple(ratings[3:6])
# Is this a fair match?
required_fairness = min(tries_left / limit, MIN_REQ_FAIRNESS)
if trueskill.quality([blue, orange]) >= required_fairness:
print(
f"Match: {picked[0:3]} vs {picked[3:6]}\nMatch quality: {trueskill.quality([blue, orange])}"
)
ticket_sys.choose(picked, bot_ids)
return picked[0:3], picked[3:6]
raise Exception("Failed to find a fair match")
def make_overlay(ld: LeagueDir, match: MatchDetails, bots: Mapping[BotID, BotConfigBundle]):
"""
Make a `current_match.json` file which contains the details about the current
match and its participants.
"""
retired = load_retired_bots(ld)
rankings = RankingSystem.load(ld).ensure_all(list(bots.keys()))
rank_list = rankings.as_sorted_list(exclude=retired)
def bot_data(bot_id):
config = bots[bot_id]
rank, mmr = [(i + 1, mrr) for i, (id, mrr, sigma) in enumerate(rank_list) if id == bot_id][0]
return {
"name": config.name,
"config_path": str(config.config_path),
"logo_path": try_copy_logo(config),
"developer": config.base_agent_config.get("Details", "developer"),
"description": config.base_agent_config.get("Details", "description"),
"fun_fact": config.base_agent_config.get("Details", "fun_fact"),
"github": config.base_agent_config.get("Details", "github"),
"language": config.base_agent_config.get("Details", "language"),
"rank": rank,
"mmr": mmr,
}
overlay = {
"blue": [bot_data(bot_id) for bot_id in match.blue],
"orange": [bot_data(bot_id) for bot_id in match.orange],
"map": match.map
}
with open(PackageFiles.overlay_current_match, 'w') as f:
json.dump(overlay, f, indent=4)
def create_bot_summary(ld: LeagueDir):
"""
Create a json file with all information about the bots. Useful for casters.
"""
bots = load_all_bots(ld)
rankings = RankingSystem.load(ld).ensure_all(list(bots.keys()))
rank_list = rankings.as_sorted_list()
def bot_data(bot_id):
config = bots[bot_id]
rank, mmr = [(i + 1, mrr)
for i, (id, mrr, sigma) in enumerate(rank_list)
if id == bot_id][0]
return {
"name": config.name,
"developer": config.base_agent_config.get("Details", "developer"),
"description":
config.base_agent_config.get("Details", "description"),
"fun_fact": config.base_agent_config.get("Details", "fun_fact"),
"github": config.base_agent_config.get("Details", "github"),
"language": config.base_agent_config.get("Details", "language"),
"rank": rank,
"mmr": mmr,
}
bot_summary = {
defmt_bot_name(bot_id): bot_data(bot_id)
for bot_id in bots.keys()
}
with open(ld.bot_summary, 'w') as f:
json.dump(bot_summary, f, indent=4)
def decide_on_players_2(bot_ids: Iterable[BotID], rank_sys: RankingSystem,
ticket_sys: TicketSystem) -> Tuple[List[BotID], List[BotID]]:
"""
Find two balanced teams. The TicketSystem and the RankingSystem to find
a fair match up between some bots that haven't played for a while.
"""
# Composing a team of the best player + the worst two players will likely yield a balanced match (0, 4, 5).
# These represent a few arrangements like that which seem reasonable to try, they will be checked against
# the trueskill system.
likely_balances = [(0, 4, 5), (0, 3, 5), (0, 2, 5), (0, 3, 4)]
# Experimental average quality based on limit:
# 1000: 0.4615
# 400: 0.460
# 100: 0.457
# 10: 0.448
num_bot_groups_to_test = 400
# How much we value the tightness of rating distribution in a given match.
# A higher number will yield matches with similarly skilled bots, but potentially lower probability of a draw.
tightness_weight = 1.0
tries_left = num_bot_groups_to_test
best_quality_found = 0
best_score_found = 0
best_match = None
chosen_balance = None
while tries_left > 0:
tries_left -= 1
# Pick some bots that haven't played for a while
picked = ticket_sys.pick_bots(bot_ids)
candidates = [Candidate(bot, rank_sys.get(bot)) for bot in picked]
candidates.sort(key=lambda c: float(c.rating), reverse=True)
tightness = 1 / (numpy.std([float(c.rating) for c in candidates]) + 1)
for balance in likely_balances:
blue_candidates = candidates[balance[0]], candidates[balance[1]], candidates[balance[2]]
orange_candidates = [c for c in candidates if c not in blue_candidates]
quality = trueskill.quality([[c.rating for c in blue_candidates], [c.rating for c in orange_candidates]])
score = quality + tightness * tightness_weight
if score > best_score_found:
best_score_found = score
best_quality_found = quality
best_match = (blue_candidates, orange_candidates)
chosen_balance = balance
blue_ids = [c.bot_id for c in best_match[0]]
orange_ids = [c.bot_id for c in best_match[1]]
tickets_consumed = sum([ticket_sys.get_ensured(b) for b in blue_ids + orange_ids])
print(f"Match: {blue_ids} vs {orange_ids}\nMatch quality: {best_quality_found} score: {best_score_found} "
f"Rank pattern: {chosen_balance}")
ticket_sys.choose(blue_ids + orange_ids, bot_ids)
return blue_ids, orange_ids
def test_gamecount_equity(self):
rank_sys = RankingSystem.read(RESOURCES_FOLDER /
'20210925212802_rankings.json')
ticket_sys = TicketSystem.read(
RESOURCES_FOLDER / '20210925212802_tickets.json', LeagueSettings())
bot_ids = rank_sys.ratings.keys()
for i in range(19):
MatchMaker.decide_on_players_2(bot_ids, rank_sys, ticket_sys)
game_counts = list(ticket_sys.session_game_counts.values())
print(ticket_sys.session_game_counts)
print(
f'Game count std dev: {numpy.std(game_counts)}, max: {max(game_counts)} min: {min(game_counts)} avg: {numpy.mean(game_counts)}'
)
for i in range(5):
print(f'Num with {i}: {game_counts.count(i)}')
def parse_subcommand_rank(args: List[str]):
assert args[0] == "rank"
help_msg = """Usage:
autoleague rank list Print list of the current leaderboard"""
ld = require_league_dir()
if len(args) == 1 or args[1] == "help":
print(help_msg)
elif args[1] == "list" and len(args) == 2:
bots = load_all_bots(ld)
rank_sys = RankingSystem.load(ld)
rank_sys.ensure_all(list(bots.keys()))
rank_sys.print_ranks_and_mmr()
else:
print(help_msg)
def parse_subcommand_rank(args: List[str]):
assert args[0] == "rank"
help_msg = """Usage:
autoleague rank list [showRetired] Print list of the current leaderboard"""
ld = require_league_dir()
if len(args) == 1 or args[1] == "help":
print(help_msg)
elif args[1] == "list" and (len(args) == 2 or len(args) == 3):
show_retired = len(args) == 3 and bool(args[2])
exclude = [] if show_retired else load_retired_bots(ld)
bots = load_all_bots(ld)
rank_sys = RankingSystem.load(ld)
rank_sys.ensure_all(list(bots.keys()))
rank_sys.print_ranks_and_mmr(exclude)
else:
print(help_msg)
def _test_decide(self, decide_name, decide_function):
rank_sys = RankingSystem.read(RESOURCES_FOLDER /
'20210925212802_rankings.json')
ticket_sys = TicketSystem.read(
RESOURCES_FOLDER / '20210925212802_tickets.json', LeagueSettings())
bot_ids = sorted(rank_sys.ratings.keys(),
key=lambda id: rank_sys.ratings[id].mu)
qualities = []
max_mmr_diffs = []
matches_played = {bot_id: 0 for bot_id in bot_ids}
for i in range(NUM_ITERATIONS):
players = decide_function(bot_ids, rank_sys, ticket_sys)
quality = get_trueskill_quality(players, rank_sys)
qualities.append(quality)
max_mmr_diff = get_max_mmr_diff(players, rank_sys)
max_mmr_diffs.append(max_mmr_diff)
for bot_id in itertools.chain.from_iterable(players):
matches_played[bot_id] += 1
average = sum(qualities) / NUM_ITERATIONS
print(f'{decide_name} average quality: {average}')
plt.hist(qualities, bins=20, range=(0, 1))
plt.ylabel('matches')
plt.xlabel('quality')
plt.title(f'{decide_name} - Matches played by quality')
plt.show()
plt.hist(max_mmr_diffs, bins=10, range=(0, 100))
plt.ylabel('matches')
plt.xlabel('max MMR difference')
plt.title(f'{decide_name} - Matches played by MMR difference')
plt.show()
matches = matches_played.values()
plt.bar(range(len(matches)), matches)
plt.ylabel('matches')
plt.xlabel('bot (sorted by MMR)')
plt.title(f'{decide_name} - Matches played by bot')
plt.show()
def parse_subcommand_bot(args: List[str]):
assert args[0] == "bot"
help_msg = """Usage:
autoleague bot list Print list of all known bots
autoleague bot test <bot_id> Run test match using a specific bot
autoleague bot details <bot_id> Print details about the given bot
autoleague bot unzip Unzip all bots in the bot directory
autoleague bot summary Create json file with bot descriptions"""
ld = require_league_dir()
if len(args) == 1 or args[1] == "help":
print(help_msg)
elif args[1] == "list" and len(args) == 2:
bot_configs = load_all_bots(ld)
rank_sys = RankingSystem.load(ld)
ticket_sys = TicketSystem.load(ld)
bot_ids = list(
set(bot_configs.keys()).union(set(rank_sys.ratings.keys())).union(
set(ticket_sys.tickets.keys())))
print(f"{'': <22} c r t")
for bot in sorted(bot_ids):
c = "x" if bot in bot_configs else " "
r = "x" if bot in rank_sys.ratings else " "
t = "x" if bot in ticket_sys.tickets else " "
print(f"{bot + ' ':.<22} {c} {r} {t}")
elif args[1] == "test" and len(args) == 3:
# Load
bots = load_all_bots(ld)
bot = args[2]
if bot not in bots:
print(f"Could not find the config file of '{bot}'")
return
# Run
match = MatchMaker.make_test_match(bot)
run_match(ld, match, bots, ReplayPreference.NONE)
print(f"Test of '{bot}' complete")
elif args[1] == "details" and len(args) == 3:
bots = load_all_bots(ld)
bot = args[2]
if bot not in bots:
print(f"Could not find the config file of '{bot}'")
return
print_details(bots[bot])
elif args[1] == "unzip" and len(args) == 2:
print("Unzipping all bots:")
unzip_all_bots(ld)
elif args[1] == "summary" and len(args) == 2:
create_bot_summary(ld)
print("Bot summary created")
else:
print(help_msg)
def get_max_mmr_diff(players: Tuple[List[BotID], List[BotID]],
rank_sys: RankingSystem) -> float:
mus = [rank_sys.get(bot).mu for bot in players[0] + players[1]]
return max(mus) - min(mus)
def get_trueskill_quality(players: Tuple[List[BotID], List[BotID]],
rank_sys: RankingSystem) -> float:
blue_ratings = [rank_sys.get(bot) for bot in players[0]]
orange_ratings = [rank_sys.get(bot) for bot in players[1]]
return trueskill.quality([blue_ratings, orange_ratings])
import numpy as np
import matplotlib.pylab as plt
from pathlib import Path
from matplotlib.colors import ListedColormap
from match import MatchDetails
from paths import LeagueDir
from ranking_system import RankingSystem
from settings import PersistentSettings
settings = PersistentSettings.load()
ld = LeagueDir(Path(settings.league_dir_raw))
ranks = RankingSystem.latest(ld, 1)[0]
bots = sorted(ranks.ratings.keys(), key=lambda bot: -ranks.get_mmr(bot))
N = len(bots)
matches = MatchDetails.all(ld)
win_rate = np.full((N, N), -0.01)
for i in range(N):
for j in range(N):
if i < j:
wins_i = 0
wins_j = 0
for match in matches:
if bots[i] in match.blue and bots[j] in match.orange:
if match.result.blue_goals > match.result.orange_goals:
wins_i += 1
else:
wins_j += 1
def make_summary(ld: LeagueDir, count: int):
"""
Make a summary of the N latest matches and the resulting ranks and tickets.
If N is 0 the summary will just contain the current ratings.
"""
summary = {}
tickets = TicketSystem.load(ld)
# ========== Matches ==========
matches = []
bot_wins = defaultdict(list) # Maps bots to list of booleans, where true=win and false=loss
if count > 0:
latest_matches = MatchDetails.latest(ld, count)
for i, match in enumerate(latest_matches):
matches.append({
"index": i,
"blue_names": [defmt_bot_name(bot_id) for bot_id in match.blue],
"orange_names": [defmt_bot_name(bot_id) for bot_id in match.orange],
"blue_goals": match.result.blue_goals,
"orange_goals": match.result.orange_goals,
})
for bot in match.blue:
bot_wins[bot].append(match.result.blue_goals > match.result.orange_goals)
for bot in match.orange:
bot_wins[bot].append(match.result.blue_goals < match.result.orange_goals)
summary["matches"] = matches
# ========= Ranks/Ratings =========
bots = load_all_unretired_bots(ld)
retired = load_retired_bots(ld)
bots_by_rank = []
if count <= 0:
# Old rankings and current rankings is the same, but make sure all bots have a rank currently
old_rankings = RankingSystem.load(ld).as_sorted_list(exclude=retired)
cur_rankings = RankingSystem.load(ld).ensure_all(list(bots.keys())).as_sorted_list(exclude=retired)
else:
# Determine current rank and their to N matches ago
n_rankings = RankingSystem.latest(ld, count + 1)
old_rankings = n_rankings[0].as_sorted_list(exclude=retired)
cur_rankings = n_rankings[-1].ensure_all(list(bots.keys())).as_sorted_list(exclude=retired)
for i, (bot, mrr, sigma) in enumerate(cur_rankings):
cur_rank = i + 1
old_rank = None
old_mmr = None
for j, (other_bot, other_mrr, _) in enumerate(old_rankings):
if bot == other_bot:
old_rank = j + 1
old_mmr = other_mrr
break
bots_by_rank.append({
"bot_id": defmt_bot_name(bot),
"mmr": mrr,
"old_mmr": old_mmr,
"sigma": sigma,
"cur_rank": cur_rank,
"old_rank": old_rank,
"tickets": tickets.get(bot) or tickets.new_bot_ticket_count,
"wins": bot_wins[bot],
})
summary["bots_by_rank"] = bots_by_rank
# =========== Write =============
with open(PackageFiles.overlay_summary, 'w') as f:
json.dump(summary, f, indent=4)
league_settings = LeagueSettings.load(ld)
league_settings.last_summary = count
league_settings.save(ld)
def parse_subcommand_retirement(args: List[str]):
assert args[0] == "retirement"
help_msg = """Usage:
autoleague retirement list Print all bots in retirement
autoleague retirement retire <bot> Retire a bot, removing it from play and the leaderboard
autoleague retirement unretire <bot> Unretire a bot
autoleague retirement retireall Retire all bots"""
ld = require_league_dir()
if len(args) == 1 or args[1] == "help":
print(help_msg)
elif args[1] == "list" and len(args) == 2:
retired = load_retired_bots(ld)
if len(retired) == 0:
print("There are no bots in retirement")
else:
print("Retired bots:")
for bot_id in sorted(retired):
print(bot_id)
elif args[1] == "retire" and len(args) == 3:
bot = args[2]
retired = load_retired_bots(ld)
retired.add(bot)
save_retired_bots(ld, retired)
print(f"Retired {bot}")
elif args[1] == "unretire" and len(args) == 3:
bot = args[2]
retired = load_retired_bots(ld)
try:
retired.remove(bot)
save_retired_bots(ld, retired)
print(f"Unretired {bot}")
except KeyError:
print(f"The bot {bot} is not in retirement")
elif args[1] == "retireall" and len(args) == 2:
bot_configs = load_all_bots(ld)
rank_sys = RankingSystem.load(ld)
ticket_sys = TicketSystem.load(ld)
retired = load_retired_bots(ld)
all_bots = set(
bot_configs.keys()).union(set(rank_sys.ratings.keys())).union(
set(ticket_sys.tickets.keys())).union(retired)
save_retired_bots(ld, all_bots)
count = len(all_bots) - len(retired)
print(f"Retired {count} bots")
else:
print(help_msg)
from paths import LeagueDir
from ranking_system import RankingSystem
from settings import PersistentSettings
import seaborn as sns
import pandas as pd
import matplotlib.pylab as plt
settings = PersistentSettings.load()
ld = LeagueDir(Path(settings.league_dir_raw))
rankings = {}
for path in list(ld.rankings.iterdir()):
time = path.name[:8]
if time not in rankings:
rankings[time] = {}
ranking = RankingSystem.read(path)
rankings[time].update(ranking.get_mmr_all())
bots = sorted(set([bot_id for time in rankings for bot_id in rankings[time]]))
times = sorted(rankings.keys())
data = {bot: [rankings[time].get(bot) or 33 for time in times] for bot in bots}
df = pd.DataFrame.from_dict(data,
orient='index',
columns=range(1,
len(times) + 1)).transpose()
print(df)
plt.figure(figsize=(10.0, 5.0))
plt.plot(df)
plt.legend(bots,
def parse_subcommand_match(args: List[str]):
assert args[0] == "match"
help_msg = """Usage:
autoleague match run Run a standard 3v3 soccer match
autoleague match undo Undo the last match
autoleague match list [n] Show the latest matches"""
ld = require_league_dir()
if len(args) == 1 or args[1] == "help":
print(help_msg)
elif args[1] == "run" and len(args) == 2:
# Load
bots = load_all_bots(ld)
rank_sys = RankingSystem.load(ld)
ticket_sys = TicketSystem.load(ld)
# Run
match = MatchMaker.make_next(bots, rank_sys, ticket_sys)
result, replay = run_match(ld, match, bots, ReplayPreference.SAVE)
rank_sys.update(match, result)
match.result = result
match.replay_id = replay.replay_id
# Save
match.save(ld)
rank_sys.save(ld, match.time_stamp)
ticket_sys.save(ld, match.time_stamp)
# Print new ranks
rank_sys.print_ranks_and_mmr()
# Make summary
league_settings = LeagueSettings.load(ld)
make_summary(ld, league_settings.last_summary + 1)
print(
f"Created summary of the last {league_settings.last_summary + 1} matches."
)
elif args[1] == "undo" and len(args) == 2:
# Undo latest match
ld = require_league_dir()
latest_matches = MatchDetails.latest(ld, 1)
if len(latest_matches) == 0:
print("No matches to undo")
else:
latest_match = latest_matches[0]
# Prompt user
print(f"Latest match was {latest_match.name}")
if prompt_yes_no(
"Are you sure you want to undo the latest match?"):
# Undo latest update to all systems
RankingSystem.undo(ld)
TicketSystem.undo(ld)
MatchDetails.undo(ld)
# New latest match
new_latest_match = MatchDetails.latest(ld, 1)
if new_latest_match:
print(f"Reverted to {new_latest_match[0].name}")
else:
print("Reverted to beginning of league (no matches left)")
elif args[1] == "list" and len(args) <= 3:
count = 999999
if len(args) == 3:
count = int(args[2])
# Show list of latest n matches played
latest_matches = MatchDetails.latest(ld, count)
if len(latest_matches) == 0:
print("No matches have been played yet.")
else:
print(f"Match history (latest {len(latest_matches)} matches):")
for match in latest_matches:
print(
f"{match.time_stamp}: {', '.join(match.blue) + ' ':.<46} {match.result.blue_goals} VS {match.result.orange_goals} {' ' + ', '.join(match.orange):.>46}"
)
else:
print(help_msg)
def main():
RankingSystem.setup()
parse_args(sys.argv[1:])
return 0
def decide_on_players_3(bot_ids: Iterable[BotID], rank_sys: RankingSystem,
ticket_sys: TicketSystem) -> Tuple[List[BotID], List[BotID]]:
"""
Find two balanced teams. The TicketSystem and the RankingSystem to find
a fair match up between some bots that haven't played for a while.
"""
# Higher ticket strength produces a more uniform distribution of matches played, adjust by increments of 0.1
TICKET_STRENGTH = 1
# Higher MMR tolerance allows accurately rated bots to play in more "distant" MMR matches, adjust by increments of 1
MMR_TOLERANCE = 4
# Max attempts to build match of quality >= MIN_QUALITY
MAX_ITERATIONS = 20
MIN_QUALITY = 0.4
rank_sys.ensure_all(bot_ids)
ticket_sys.ensure(bot_ids)
best_quality = 0
best_match = None
max_tickets = max([ticket_sys.get(bot_id) for bot_id in bot_ids])
for i in range(MAX_ITERATIONS):
# Get Leader Bot (choose randomly between bots with highest tickets)
possible_leaders = [bot_id for bot_id, tickets in ticket_sys.tickets.items() if tickets == max_tickets and bot_id in bot_ids]
leader = numpy.random.choice(possible_leaders)
# Get MU for Leader bot, that will be the match mmr
match_mmr = rank_sys.get(leader).mu
# Score all bots based on probability to perform at target mmr, scaled by amount of tickets
candidates = [Candidate(bot_id, rank_sys.get(bot_id)) for bot_id in bot_ids if bot_id != leader]
scores = []
for c in candidates:
# Calculate probability to perform at desired mmr
performance_prob = pdf(match_mmr, mu=c.rating.mu, sigma=math.sqrt(c.rating.sigma**2 + MMR_TOLERANCE**2))
# Calculate weighting factor based on tickets
tickets = ticket_sys.get(c.bot_id)
tickets_weight = tickets ** TICKET_STRENGTH
# Calculate candidate score
scores.append(performance_prob * tickets_weight)
# Pick 5 bots randomly based on their score
probs = numpy.asarray(scores) / sum(scores)
players = list(numpy.random.choice(candidates, size=5, p=probs, replace=False))
players.append(Candidate(leader, rank_sys.get(leader)))
# Get the highest quality match with the 6 chosen bots
combinations = list(itertools.combinations(players, 3))
possible_matches = len(combinations) // 2
blue_combs = combinations[:possible_matches]
orange_combs = combinations[:possible_matches-1:-1]
for i in range(possible_matches):
blue_team = blue_combs[i]
orange_team = orange_combs[i]
quality = trueskill.quality([[c.rating for c in blue_team], [c.rating for c in orange_team]])
if quality > best_quality:
best_quality = quality
best_match = (blue_team, orange_team)
if best_quality >= MIN_QUALITY:
break
# We sort by get_mmr() because it considers sigma
blue_ids = sorted([c.bot_id for c in best_match[0]], key=lambda id: rank_sys.get_mmr(id), reverse=True)
orange_ids = sorted([c.bot_id for c in best_match[1]], key=lambda id: rank_sys.get_mmr(id), reverse=True)
tickets_consumed = sum([ticket_sys.get_ensured(b) for b in blue_ids + orange_ids])
print(f"Match: {blue_ids} vs {orange_ids}\nMatch quality: {best_quality} Tickets consumed: {tickets_consumed}")
ticket_sys.choose(blue_ids + orange_ids, bot_ids)
return blue_ids, orange_ids