def __init__(self,
policy_value_fn,
c_puct=5,
n_playout=2000,
is_selfplay=0):
self.mcts = MCTS(policy_value_fn, c_puct, n_playout)
self.is_selfplay = is_selfplay
def test_mcts_from_root_with_equal_priors(self):
class MockModel:
def predict(self, board):
# starting board is:
# [0, 0, 1, -1]
return np.array([0.26, 0.24, 0.24, 0.26]), 0.0001
game = Connect2Game()
args = {'num_simulations': 50}
model = MockModel()
mcts = MCTS(game, model, args)
canonical_board = [0, 0, 0, 0]
print("starting")
root = mcts.run(model,
canonical_board,
to_play=1,
add_exploration_noise=False)
# the best move is to play at index 1 or 2
best_outer_move = max(root.children[0].visit_count,
root.children[0].visit_count)
best_center_move = max(root.children[1].visit_count,
root.children[2].visit_count)
self.assertGreater(best_center_move, best_outer_move)
def exceute_episode(self):
train_examples = []
current_player = 1
state = self.game.get_init_board()
while True:
canonical_board = self.game.get_canonical_board(
state, current_player)
self.mcts = MCTS(self.game, self.model, self.args)
root = self.mcts.run(self.model, canonical_board, to_play=1)
action_probs = [0 for _ in range(self.game.get_action_size())]
for k, v in root.children.items():
action_probs[k] = v.visit_count
action_probs = action_probs / np.sum(action_probs)
train_examples.append(
(canonical_board, current_player, action_probs))
action = root.select_action(temperature=0)
state, current_player = self.game.get_next_state(
state, current_player, action)
reward = self.game.get_reward_for_player(state, current_player)
if reward is not None:
ret = []
for hist_state, hist_current_player, hist_action_probs in train_examples:
# [Board, currentPlayer, actionProbabilities, Reward]
ret.append(
(hist_state, hist_action_probs, reward *
((-1)**(hist_current_player != current_player))))
return ret
def play_game():
tree = MCTS()
board = new_domineering_board()
board.to_pretty_string()
while True:
row_col = input("enter row,col: ")
row, col = map(int, row_col.split(","))
stdout.write('You choose ({}, {})'.format(row, col))
index = conf.BOARD_Y_SIZE * (row - 1) + (col - 1)
if (board.tup[index] is not None) and (
board.is_valid_move(index + conf.BOARD_Y_SIZE)):
raise RuntimeError("Invalid move")
board = board.make_move(index)
board.to_pretty_string()
if board.terminal:
stdout.write("\nWinner is {}".format(
conf.PLAYERS_NAME[board.winner]))
break
# You can train as you go, or only at the beginning.
# Here, we train as we go, doing fifty rollouts each turn.
for _ in range(conf.TRAINING_EPOCHS):
tree.do_rollout(board)
board = tree.choose(board)
board.to_pretty_string()
if board.terminal:
stdout.write("\nWinner is {}".format(
conf.PLAYERS_NAME[board.winner]))
break
def play_game():
tree = MCTS()
board = new_tic_tac_toe_board()
print(board.to_pretty_string())
while True:
row_col = input("enter row,col: ")
row, col = map(int, row_col.split(","))
index = 3 * (row - 1) + (col - 1)
if board.tup[index] is not None:
raise RuntimeError("Invalid move")
board = board.make_move(index)
print(board.to_pretty_string())
if board.terminal:
break
# You can train as you go, or only at the beginning.
# Here, we train as we go, doing fifty rollouts each turn.
for _ in range(2):
tree.do_rollout(board)
print(tree.children)
print(len(tree.children[board]))
for b in tree.children[board]:
print(colored(b.to_pretty_string(), 'green'))
board = tree.choose(board)
print(board.to_pretty_string())
if board.terminal:
break
def play_game_ocba(budget=1000, optimum=0, n0=5, sigma_0=1):
mcts = MCTS(policy='ocba',
budget=budget,
optimum=optimum,
n0=n0,
sigma_0=sigma_0)
tree = new_tree()
for _ in range(budget):
mcts.do_rollout(tree)
next_tree = mcts.choose(tree)
return (mcts, tree, next_tree)
class MCTSAI(AI):
tree = MCTS()
def __init__(self, name: str, nRollout: int = 5):
self.nRollout: int = nRollout
super().__init__(name)
def play(self, state: ReversiState):
for _ in range(self.nRollout):
self.tree.do_rollout(state)
return self.tree.choose(state)
to_char = lambda v: ("⚫" if v is True else ("⚪"
if v is False else " "))
def test_mcts_finds_best_move_with_equal_priors(self):
class MockModel:
def predict(self, board):
return np.array([0.51, 0.49, 0, 0]), 0.0001
game = Connect2Game()
args = {'num_simulations': 25}
model = MockModel()
mcts = MCTS(game, model, args)
canonical_board = [0, 0, -1, 1]
root = mcts.run(model, canonical_board, to_play=1)
# the better move is to play at index 1
self.assertLess(root.children[0].visit_count,
root.children[1].visit_count)
def play_game_uct(budget=1000,
exploration_weight=1,
optimum=0,
n0=2,
sigma_0=1):
mcts = MCTS(policy='uct',
exploration_weight=exploration_weight,
budget=budget,
n0=n0,
sigma_0=sigma_0)
tree = new_tree()
for _ in range(budget):
mcts.do_rollout(tree)
next_tree = mcts.choose(tree)
return (mcts, tree, next_tree)
def exceute_episode(self):
train_examples = []
current_player = 1
episode_step = 0
state = self.game.get_init_board()
while True:
episode_step += 1
canonical_board = self.game.get_canonical_board(
state, current_player)
temp = int(episode_step < self.args['tempThreshold'])
add_exploration_noise = temp > 0
self.mcts = MCTS(self.game, self.model, self.args)
root = self.mcts.run(self.model,
canonical_board,
to_play=1,
add_exploration_noise=add_exploration_noise)
action_probs = [0 for _ in range(self.game.get_action_size())]
for k, v in root.children.items():
action_probs[k] = v.visit_count
action_probs = action_probs / np.sum(action_probs)
train_examples.append(
(canonical_board, current_player, action_probs))
action = root.select_action(temp)
state, current_player = self.game.get_next_state(
state, current_player, action)
reward = self.game.get_game_ended(state, current_player)
if reward is not None:
ret = []
for hist_state, hist_current_player, hist_action_probs in train_examples:
# [Board, currentPlayer, actionProbabilities, Reward]
ret.append(
(hist_state, hist_action_probs, reward *
((-1)**(hist_current_player != current_player))))
return ret
def test_mcts_finds_best_move_with_really_bad_priors(self):
class MockModel:
def predict(self, board):
# starting board is:
# [0, 0, 1, -1]
return np.array([0.3, 0.7, 0, 0]), 0.0001
game = Connect2Game()
args = {'num_simulations': 25}
model = MockModel()
mcts = MCTS(game, model, args)
canonical_board = [0, 0, 1, -1]
print("starting")
root = mcts.run(model, canonical_board, to_play=1)
# the best move is to play at index 1
self.assertGreater(root.children[1].visit_count,
root.children[0].visit_count)
def exceute_episode(self):
train_examples = []
current_player = 1
state = gogame.init_state(self.args['boardSize'])
while True:
#print("while True")
canonical_board = gogame.canonical_form(state)
self.mcts = MCTS(self.game, self.model, self.args)
root = self.mcts.run(self.model, canonical_board, to_play=1)
action_probs = [
0 for _ in range((self.args['boardSize'] *
self.args['boardSize']) + 1)
]
for k, v in root.children.items():
action_probs[k] = v.visit_count
action_probs = action_probs / np.sum(action_probs)
train_examples.append(
(canonical_board, current_player, action_probs))
action = root.select_action(temperature=1)
state = gogame.next_state(state, action, canonical=False)
current_player = -current_player
reward = gogame.winning(
state) * current_player if gogame.game_ended(state) else None
if reward is not None:
ret = []
for hist_state, hist_current_player, hist_action_probs in train_examples:
# [Board, currentPlayer, actionProbabilities, Reward]
tfBoard = np.array(
[hist_state[0], hist_state[1],
hist_state[3]]).transpose().tolist()
#ret.append(np.array([tfBoard,tfBoard, hist_action_probs, reward * ((-1) ** (hist_current_player != current_player))]))
ret.append(
(tfBoard, hist_action_probs, reward *
((-1)**(hist_current_player != current_player))))
return ret
def mcts_playout(depth, num_iter, num_rollout, exploration_weight):
root, leaf_nodes_dict = make_binary_tree(depth=depth)
leaf_nodes_dict_sorted = sorted(leaf_nodes_dict.items(),
key=lambda x: x[1],
reverse=True)
print("Expected (max) leaf node: {}, value: {}".format(
leaf_nodes_dict_sorted[0][0], leaf_nodes_dict_sorted[0][1]))
print("Expected (min) leaf node: {}, value: {}".format(
leaf_nodes_dict_sorted[-1][0], leaf_nodes_dict_sorted[-1][1]))
mcts = MCTS(exploration_weight=exploration_weight)
while True:
# we run MCTS simulation for many times
for _ in range(num_iter):
mcts.run(root, num_rollout=num_rollout)
# we choose the best greedy action based on simulation results
root = mcts.choose(root)
# we repeat until root is terminal
if root.is_terminal():
print("Found optimal (max) leaf node: {}, value: {}".format(
root, root.value))
return root.value
def test_mcts_finds_best_move_with_really_really_bad_priors(self):
class MockModel:
def predict(self, board):
# starting board is:
# [-1, 0, 0, 0]
return np.array([0, 0.3, 0.3, 0.3]), 0.0001
game = Connect2Game()
args = {'num_simulations': 100}
model = MockModel()
mcts = MCTS(game, model, args)
canonical_board = [-1, 0, 0, 0]
root = mcts.run(model,
canonical_board,
to_play=1,
add_exploration_noise=False)
# the best move is to play at index 1
self.assertGreater(root.children[1].visit_count,
root.children[2].visit_count)
self.assertGreater(root.children[1].visit_count,
root.children[3].visit_count)
def __init__(self, game, model, args):
self.game = game
self.model = model
self.args = args
self.mcts = MCTS(self.game, self.model, self.args)
def __init__(self, _id):
super().__init__(_id)
self.opponent_id = None
self.tree = MCTS()