def start(): config = RankingConfig() print("Merging Staged Files") df = util.mergeStagedWTL(config) if df is None: print("Issue loading file, will keep trying") print("If this error doesn't go away soon, you may need to delete %s"% os.path.normpath(config.data.performance_location+"temp_win_matrix.csv")) print("You could also try manually merging it with %s"% os.path.normpath(config.data.performance_location+"win_matrix.csv")) while df is None: time.sleep(0.1) df = util.mergeStagedWTL(config) players = list(df) print("\n",players) win_matrix = df.as_matrix() print("\nTotal Games: ", np.sum(win_matrix)) print("\nWin Matrix(row beat column):") print(win_matrix) try: with np.errstate(divide='ignore', invalid='ignore'): params = choix.ilsr_pairwise_dense(win_matrix) print("\nRankings:") for i, player in enumerate(np.argsort(params)[::-1]): print("%d. %s with %0.2f rating"% (i+1, players[player], params[player])) print("\n(Rating Diff, Winrate) -> (0.5, 62%), (1, 73%), (2, 88%), (3, 95%), (5, 99%)") except Exception: print("\nNot Enough data to calculate rankings")
def getKingIndex(win_matrix): try: with np.errstate(divide='ignore', invalid='ignore'): params = choix.ilsr_pairwise_dense(win_matrix) return np.argsort(params)[-1] except Exception: return -1
def getRankings(win_matrix): try: with np.errstate(divide='ignore', invalid='ignore'): params = choix.ilsr_pairwise_dense(win_matrix) return np.argsort(params).tolist() except Exception: return []
def start(): config = RankingConfig() print("Merging Staged Files") df = util.mergeStagedWTL(config) if df is None: print("Issue loading file, will keep trying") print("If this error doesn't go away soon, you may need to delete %s" % os.path.normpath(config.data.performance_location + "temp_win_matrix.csv")) print("You could also try manually merging it with %s" % os.path.normpath(config.data.performance_location + "win_matrix.csv")) while df is None: time.sleep(0.1) df = util.mergeStagedWTL(config) players = list(df) print("\n", players) win_matrix = df.as_matrix() print("\nTotal Games: ", np.sum(win_matrix)) print("\nWin Matrix(row beat column):") print(win_matrix) try: with np.errstate(divide='ignore', invalid='ignore'): params = choix.ilsr_pairwise_dense(win_matrix) print("\nRankings:") for i, player in enumerate(np.argsort(params)[::-1]): print("%d. %s with %0.2f rating" % (i + 1, players[player], params[player])) print( "\n(Rating Diff, Winrate) -> (0.5, 62%), (1, 73%), (2, 88%), (3, 95%), (5, 99%)" ) except Exception: print("\nNot Enough data to calculate rankings")
def calc_ranking(config): models = sorted(glob.glob(config.data.model_location + "*.h5")) players = [] for i, model in enumerate(models): if i % config.model_skip == 0 or i == len(models): players.append(model) wtl = np.zeros((len(players), 3)) win_matrix = np.zeros((len(players), len(players))) game = Othello() king_index = len(players) - 1 king = AIPlayer(0, config.game.simulation_num_per_move, train=False, model=players[king_index], tau=config.game.tau_1) challenger = AIPlayer(0, config.game.simulation_num_per_move, train=False, model=players[0], tau=config.game.tau_1) total_games = (config.game_num_per_model * (len(players))) // 2 played_games = 0 start = time() print("Playing king of the hill with %d players and %d games per player" % (len(players), config.game_num_per_model)) if config.game_num_per_model < len(players): print( "We suggest that you increase games per player to be greater than players" ) for i in range(math.ceil(total_games / (len(players) - 1))): AIPlayer.clear() king_index = getKingIndex(win_matrix) if king_index == -1: king_index = (len(players) - 1) - i % len(players) msg = "No King Yet" else: msg = "King is " + os.path.basename( players[king_index]).split(".")[0] king.load(players[king_index]) if config.print_king: print(msg.ljust(90)) for j in range(len(players)): util.print_progress_bar(played_games, total_games, start=start) if j == king_index: continue challenger.load(players[j]) if random.random() < 0.5: king_side = -1 p1 = king p2 = challenger else: king_side = 1 p1 = challenger p2 = king side = -1 turn = 1 while not game.game_over(): tau = config.game.tau_1 if config.game.tau_swap < turn: tau = config.game.tau_2 p1.tau = tau p2.tau = tau if side == -1: t = p1.pick_move(game, side) else: t = p2.pick_move(game, side) game.play_move(t[0], t[1], side) side *= -1 turn += 1 if game.get_winner() == king_side: win_matrix[king_index, j] += 1 wtl[king_index, 0] += 1 wtl[j, 2] += 1 elif game.get_winner() == -1 * king_side: win_matrix[j, king_index] += 1 wtl[king_index, 2] += 1 wtl[j, 0] += 1 else: win_matrix[king_index, j] += 0.5 win_matrix[j, king_index] += 0.5 wtl[king_index, 1] += 1 wtl[j, 1] += 1 game.reset_board() played_games += 1 if played_games == total_games: break util.print_progress_bar(total_games, total_games, start=start) try: params = choix.ilsr_pairwise_dense(win_matrix) print("\nRankings:") for i, player in enumerate(np.argsort(params)[::-1]): print( "%d. %s (expected %d) with %0.2f rating and results of %d-%d-%d" % (i + 1, os.path.basename(players[player]).split(".")[0], len(players) - player, params[player], wtl[player, 0], wtl[player, 1], wtl[player, 2])) print( "\n(Rating Diff, Winrate) -> (0.5, 62%), (1, 73%), (2, 88%), (3, 95%), (5, 99%)" ) except Exception: print("\n Not Enough data to calculate rankings") print("\nWin Matrix:") print(win_matrix) print("\nResults:") for player in range(win_matrix.shape[0]): print("%s results of %d-%d-%d" % (os.path.basename(players[player]).split(".")[0], wtl[player, 0], wtl[player, 1], wtl[player, 2]))
def calc_ranking(config): models = sorted(glob.glob(config.data.model_location + "*.h5")) players = [] for i, model in enumerate(models): if i % config.model_skip == 0 or i == len(models): players.append(model) wtl = np.zeros((len(players), 3)) win_matrix = np.zeros((len(players), len(players))) game = Othello() ##give every player a random order to play games against opponents order = [] for i in range(len(players)): nums = [x for x in range(len(players))] nums.remove(i) random.shuffle(nums) order.append(nums) p1 = AIPlayer(1, config.game.simulation_num_per_move, model=players[0]) p2 = AIPlayer(1, config.game.simulation_num_per_move, model=players[order[0][0]]) start = time() print( "Playing random round robin with %d players and %d games per player" % (len(players), config.game_num_per_model)) for i in range(config.game_num_per_model // 2): util.print_progress_bar(i, config.game_num_per_model // 2, start=start) ordering = [x for x in range(len(players))] random.shuffle(ordering) for j in ordering: AIPlayer.clear() x = i if x >= len(order[j]): x %= len(order[j]) if x == 0: random.shuffle(order[j]) p1.load(players[j]) p2.load(players[order[j][x]]) side = -1 turn = 1 while not game.game_over(): tau = config.game.tau_1 if config.game.tau_swap < turn: tau = config.game.tau_2 p1.tau = tau p2.tau = tau if side == -1: t = p1.pick_move(game, side) else: t = p2.pick_move(game, side) game.play_move(t[0], t[1], side) side *= -1 turn += 1 if game.get_winner() == -1: win_matrix[j, order[j][x]] += 1 wtl[j, 0] += 1 wtl[order[j][x], 2] += 1 elif game.get_winner() == 1: win_matrix[order[j][x], j] += 1 wtl[j, 2] += 1 wtl[order[j][x], 0] += 1 else: win_matrix[j, order[j][x]] += 0.5 win_matrix[order[j][x], j] += 0.5 wtl[j, 1] += 1 wtl[order[j][x], 1] += 1 game.reset_board() util.print_progress_bar(config.game_num_per_model // 2, config.game_num_per_model // 2, start=start) params = choix.ilsr_pairwise_dense(win_matrix) print("\nRankings:") for i, player in enumerate(np.argsort(params)[::-1]): print( "%d. %s (expected %d) with %0.2f rating and results of %d-%d-%d" % (i + 1, os.path.basename(players[player]), len(players) - player, params[player], wtl[player, 0], wtl[player, 1], wtl[player, 2])) print( "\n(Rating Diff, Winrate) -> (0.5, 62%), (1, 73%), (2, 88%), (3, 95%), (5, 99%)" )
def calc_ranking(config): models = sorted(glob.glob(config.data.model_location+"*.h5")) players = [] for i, model in enumerate(models): if i % config.model_skip == 0 or i == len(models): players.append(model) wtl = np.zeros((len(players), len(players), 3)) win_matrix = np.zeros((len(players),len(players))) game = Othello() challenger1 = AIPlayer(0, config.game.simulation_num_per_move, train=False, model=players[-1], tau=config.game.tau_1) challenger2 = AIPlayer(0, config.game.simulation_num_per_move, train=False, model=players[0], tau=config.game.tau_1) total_games = (config.game_num_per_model * (len(players)))//2 played_games = 0 finished = False start = time() print("Ranking with %d players and %d games per player" % (len(players), config.game_num_per_model)) if config.game_num_per_model < len(players): print("We suggest that you increase games per player to be greater than players") for i in itertools.count(): ranks = getRankings(win_matrix) if len(ranks) == 0: msg = "No Clear Best Yet" else: msg = "Current Best is "+util.getPlayerName(players[ranks[-1]]) if config.print_best: print(msg.ljust(90)) for j in range(len(players)): util.print_progress_bar(played_games, total_games, start=start) challenger1_index = getLeastPlayed(win_matrix, j) AIPlayer.clear() challenger1.load(players[challenger1_index]) challenger2.load(players[j]) if random.random() < 0.5: challenger1_side = -1 p1 = challenger1 p2 = challenger2 else: challenger1_side = 1 p1 = challenger2 p2 = challenger1 side = -1 turn = 1 while not game.game_over(): tau = config.game.tau_1 if config.game.tau_swap < turn: tau = config.game.tau_2 p1.tau = tau p2.tau = tau if side == -1: t = p1.pick_move(game, side) else: t = p2.pick_move(game, side) game.play_move(t[0], t[1], side) side *= -1 turn += 1 if game.get_winner() == challenger1_side: win_matrix[challenger1_index,j] += 1 wtl[challenger1_index, j,0] += 1 elif game.get_winner() == -1*challenger1_side: win_matrix[j, challenger1_index] += 1 wtl[challenger1_index, j,2] += 1 else: win_matrix[challenger1_index,j] += 0.5 win_matrix[j, challenger1_index] += 0.5 wtl[challenger1_index, j, 1] += 1 game.reset_board() played_games += 1 if played_games >= total_games: finished = True break saveWTL(config, players, wtl) wtl = np.zeros((len(players), len(players), 3)) if finished: break util.print_progress_bar(total_games, total_games, start=start) print("\n",[util.getPlayerName(player) for player in players]) print("\nWin Matrix(row beat column):") print(win_matrix) try: with np.errstate(divide='ignore', invalid='ignore'): params = choix.ilsr_pairwise_dense(win_matrix) print("\nRankings:") for i, player in enumerate(np.argsort(params)[::-1]): print("%d. %s (expected %d) with %0.2f rating"% (i+1, util.getPlayerName(players[player]), len(players)-player, params[player])) print("\n(Rating Diff, Winrate) -> (0.5, 62%), (1, 73%), (2, 88%), (3, 95%), (5, 99%)") except Exception: print("\nNot Enough data to calculate rankings")
def calc_ranking(config): models = sorted(glob.glob(config.data.model_location + "*.h5")) players = [] for i, model in enumerate(models): if i % config.model_skip == 0 or i == len(models): players.append(model) wtl = np.zeros((len(players), len(players), 3)) win_matrix = np.zeros((len(players), len(players))) game = Othello() challenger1 = AIPlayer(0, config.game.simulation_num_per_move, train=False, model=players[-1], tau=config.game.tau_1) challenger2 = AIPlayer(0, config.game.simulation_num_per_move, train=False, model=players[0], tau=config.game.tau_1) total_games = (config.game_num_per_model * (len(players))) // 2 played_games = 0 finished = False start = time() print("Ranking with %d players and %d games per player" % (len(players), config.game_num_per_model)) if config.game_num_per_model < len(players): print( "We suggest that you increase games per player to be greater than players" ) for i in itertools.count(): ranks = getRankings(win_matrix) if len(ranks) == 0: msg = "No Clear Best Yet" else: msg = "Current Best is " + util.getPlayerName(players[ranks[-1]]) if config.print_best: print(msg.ljust(90)) for j in range(len(players)): util.print_progress_bar(played_games, total_games, start=start) challenger1_index = getLeastPlayed(win_matrix, j) AIPlayer.clear() challenger1.load(players[challenger1_index]) challenger2.load(players[j]) if random.random() < 0.5: challenger1_side = -1 p1 = challenger1 p2 = challenger2 else: challenger1_side = 1 p1 = challenger2 p2 = challenger1 side = -1 turn = 1 while not game.game_over(): tau = config.game.tau_1 if config.game.tau_swap < turn: tau = config.game.tau_2 p1.tau = tau p2.tau = tau if side == -1: t = p1.pick_move(game, side) else: t = p2.pick_move(game, side) game.play_move(t[0], t[1], side) side *= -1 turn += 1 if game.get_winner() == challenger1_side: win_matrix[challenger1_index, j] += 1 wtl[challenger1_index, j, 0] += 1 elif game.get_winner() == -1 * challenger1_side: win_matrix[j, challenger1_index] += 1 wtl[challenger1_index, j, 2] += 1 else: win_matrix[challenger1_index, j] += 0.5 win_matrix[j, challenger1_index] += 0.5 wtl[challenger1_index, j, 1] += 1 game.reset_board() played_games += 1 if played_games >= total_games: finished = True break saveWTL(config, players, wtl) wtl = np.zeros((len(players), len(players), 3)) if finished: break util.print_progress_bar(total_games, total_games, start=start) print("\n", [util.getPlayerName(player) for player in players]) print("\nWin Matrix(row beat column):") print(win_matrix) try: with np.errstate(divide='ignore', invalid='ignore'): params = choix.ilsr_pairwise_dense(win_matrix) print("\nRankings:") for i, player in enumerate(np.argsort(params)[::-1]): print("%d. %s (expected %d) with %0.2f rating" % (i + 1, util.getPlayerName( players[player]), len(players) - player, params[player])) print( "\n(Rating Diff, Winrate) -> (0.5, 62%), (1, 73%), (2, 88%), (3, 95%), (5, 99%)" ) except Exception: print("\nNot Enough data to calculate rankings")
mcts2 = MCTS(g, nnet2, args) def p2(x, turn): if turn <= 2: mcts2.reset() temp = args.temp if turn <= args.tempThreshold else args.arenaTemp policy = mcts2.getActionProb(x, temp=temp) return np.random.choice(len(policy), p=policy) else: p2 = RandomPlayer(g).play arena = Arena(p1, p2, g) p1wins, p2wins, draws = arena.playGames(args.arenaCompare) win_matrix[i, j] = p1wins + 0.5*draws win_matrix[j, i] = p2wins + 0.5*draws print(f'wins: {p1wins}, ties: {draws}, losses:{p2wins}\n') print("\nWin Matrix(row beat column):") print(win_matrix) try: with np.errstate(divide='ignore', invalid='ignore'): params = choix.ilsr_pairwise_dense(win_matrix) print("\nRankings:") for i, player in enumerate(np.argsort(params)[::-1]): name = 'random' if args.playRandom and player == model_count - \ 1 else Path(networks[player]).stem print(f"{i+1}. {name} with {params[player]:0.2f} rating") print( "\n(Rating Diff, Winrate) -> (0.5, 62%), (1, 73%), (2, 88%), (3, 95%), (5, 99%)") except Exception: print("\nNot Enough data to calculate rankings")