def fight_agent(best_model: str,
current_model: str,
ae,
round_fight=AlphaZeroConfig.ROUND_ARENA,
max_turn=AlphaZeroConfig.MAX_TURN_ARENA,
max_simulation=AlphaZeroConfig.MAX_SIMULATION_ARENA):
"""
The pitted 2 agents. We will check who is the best here.
:param best_model: The current best model file path
:param current_model: The current model file path
:param ae: The Action Encoder
:param round_fight: number of round to determine the winner
:param max_turn: The maximum turn of the game. If the current turn is higher than max turn.
It will be cut and the outcome of the game is draw.
:param max_simulation: The maximum of simulation
:return: dict, The dictionary of the score
"""
from ai_modules.reinforcement_algorithm import AlphaZeroAgent
loss_win = {0: 0, 1: 0}
for round in range(round_fight):
print("ROUND {}".format(round + 1))
terminal = False
count_turn = 1
state = State()
state.initial_state()
best_model_agent = AlphaZeroAgent(state, max_simulation,
best_model) # 1
current_model_agent = None # 0
while not terminal and count_turn <= max_turn:
print("=======TURN {} ========".format(count_turn))
state.print_board()
current_player_turn = state.get_player_turn()
if current_player_turn == 1:
key, dict_key = best_model_agent.choose_action(state)
state = AIElements.result_function(state, dict_key)
if current_model_agent is not None:
current_model_agent.enemy_turn_action(key, state)
else:
if current_model_agent is None:
current_model_agent = AlphaZeroAgent(
state, max_simulation, current_model)
key, dict_key = current_model_agent.choose_action(state)
state = AIElements.result_function(state, dict_key)
best_model_agent.enemy_turn_action(key, state)
print("Player %d choose action %s" % (current_player_turn, key))
game_ended = state.is_terminal()
if game_ended:
print("Player {} Win".format(count_turn % 2))
loss_win[(current_player_turn) % 2] += 1
terminal = True
count_turn += 1
if count_turn > max_turn:
print("ROUND {} DRAW".format(round + 1))
return loss_win
def _minimax(self, current_depth, state, is_max_turn):
"""
Minimax Helper
:param current_depth: the current tree depth in the recursion
:param state: the current state in the tree node
:param is_max_turn: bool check the current's node maximizer or not?
:return:
"""
if current_depth == self.max_depth or state.is_terminal():
return AIElements.evaluation_function(state, self.player_color), ""
self.node_expanded += 1
possible_action = AIElements.get_possible_action(state)
key_of_actions = list(possible_action.keys())
shuffle(key_of_actions) #randomness
best_value = float('-inf') if is_max_turn else float('inf')
action_target = ""
for action_key in key_of_actions:
new_state = AIElements.result_function(state,
possible_action[action_key])
eval_child, action_child = self._minimax(current_depth + 1,
new_state,
not is_max_turn)
if is_max_turn and best_value < eval_child:
best_value = eval_child
action_target = action_key
elif (not is_max_turn) and best_value > eval_child:
best_value = eval_child
action_target = action_key
return best_value, action_target
def choose_action(self, state):
"""
Predict the move using AlphaZero algorithm
Parameters
----------
state : State
unused. The purpose of this parameter is to match the other class.
float, str:
The evaluation or utility and the action key name
"""
from ai_modules.ai_elements import AIElements
import numpy as np
self.mcts.self_play()
action_proba = np.array(self.mcts.get_action_proba(temperature=0))
action = np.random.choice(len(action_proba), p=action_proba)
action_key = self.ae.inverse_transform([action])[0]
possible_action = AIElements.get_possible_action(
self.stacked_state.head)
from ai_modules.ai_elements import AIElements
new_state = AIElements.result_function(self.stacked_state.head,
possible_action[action_key])
self.stacked_state.append(new_state)
self.mcts.update_root(action_key)
return (action_key, possible_action[action_key])
def _minimax(self, current_depth, state, is_max_turn, alpha, beta):
"""
Helper function of minimax
:param current_depth: The current depth on the tree in recursive
:param state: State of the current node in recursive
:param is_max_turn: Check if the current node is the max turn in recursive
:param alpha: parameter of AB Prunning, save the current maximizer best value
:param beta: parameter of AB Prunning, save the current minimizer best value
:return: int , str The value of the best action and the name of the action
"""
if current_depth == self.max_depth or state.is_terminal():
return AIElements.evaluation_function(state, self.player_color), ""
self.node_expanded += 1
possible_action = AIElements.get_possible_action(state)
key_of_actions = list(possible_action.keys())
shuffle(key_of_actions) # add randomness here
best_value = float('-inf') if is_max_turn else float('inf')
action_target = ""
for action_key in key_of_actions:
new_state = AIElements.result_function(state,
possible_action[action_key])
eval_child, action_child = self._minimax(current_depth + 1,
new_state,
not is_max_turn, alpha,
beta)
if is_max_turn and best_value < eval_child:
best_value = eval_child
action_target = action_key
alpha = max(alpha, best_value)
if beta <= alpha:
break
elif (not is_max_turn) and best_value > eval_child:
best_value = eval_child
action_target = action_key
beta = min(beta, best_value)
if beta <= alpha:
break
return best_value, action_target
def receive_input_action_play(self, input_key, input_params):
"""
Process the input from the user in the view
:param input_key: str, the key of the action
:param input_params: dict, the parameter of the action
:return: bool, tell that the action is present in the possible action
"""
if input_key in self.possible_action_keys:
self.state = AIElements.result_function(self.state, input_params)
## Useful for alpha zero only
self.ai_agent.enemy_turn_action(input_key, input_params)
index_player = (AIElements.get_player(self.state) + 1) % 2
print("TURN %d" % (self.state.turn))
print("The Evaluation of Player %d is %.2f" %
(index_player,
AIElements.evaluation_function(self.state, index_player)))
return True
else:
return False
def do_self_play_episode(
stacked_state,
mcts,
ae: ActionEncoder,
greed=False,
pov=0,
temperature_turn_end=AlphaZeroConfig.TEMPERATURE_END_STEP,
greedy_turn=AlphaZeroConfig.GREEDY_TURN_MIN,
max_turn_episode=AlphaZeroConfig.MAX_TURN_SIMULATION):
"""
Self play an episode. This function will simulate MCTS every turns.
:param stacked_state: The state that is stacked.
:param mcts: a MCTS that will be simulated every turns.
:param ae: Action Encoder
:param greed: Hack that will be used to make the agent prioritize attacking and promoting.
:param pov: The chosen point of view of player
:param temperature_turn_end: Make the temperature to 0 on the given turn
:return: Training data that will be appended to the global list of training data
"""
terminal = False
counter_step = 0
list_training = []
while not terminal:
counter_step += 1
print("Step : %d" % (counter_step))
print("Turn : %d" % (stacked_state.head.turn))
print("Player in state : %d" % (stacked_state.head.get_player_turn()))
print("Player in tree : %d" %
(mcts.root.stacked_state.head.get_player_turn()))
temperature = 1
if counter_step > temperature_turn_end:
temperature = 0
# Hack if counter_step > greedy_turn, greedily attack enemies
if counter_step > greedy_turn and greed:
print("Greedy Mode Activate")
mcts.self_play(greed)
else:
mcts.self_play()
action_proba = np.array(mcts.get_action_proba(temperature))
action = np.random.choice(len(action_proba), p=action_proba)
action_key = ae.inverse_transform([action])[0]
possible_action = AIElements.get_possible_action(stacked_state.head)
# Append list training
if stacked_state.head.get_player_turn() != pov:
training_data_tobe = HelperTrainingExample(
mirror_stacked_state(deepcopy(stacked_state)),
(stacked_state.head.get_player_turn()),
ae.array_mirrored(action_proba))
else:
training_data_tobe = HelperTrainingExample(
deepcopy(stacked_state), stacked_state.head.get_player_turn(),
action_proba)
print("Action_proba shape {}".format(action_proba.shape))
list_training.append(training_data_tobe)
# TODO : add v_treshold to cut
stacked_state.head.print_board()
print('Mirrored')
mirror_stacked_state(deepcopy(stacked_state)).head.print_board()
print("Player %d choose action %s" %
(stacked_state.head.get_player_turn(), action_key))
print("Next mean action value : %.4f" %
(mcts.root.q_state_action[action_key]))
new_state = AIElements.result_function(stacked_state.head,
possible_action[action_key])
stacked_state.append(new_state)
mcts.update_root(action_key)
terminal = stacked_state.head.is_terminal()
if terminal:
# Update the reward in list_training
mcts.self_play() # to fill the v variable
reward = mcts.root.v # reward should be -1
loser_player = stacked_state.head.get_player_turn()
for i in list_training:
if i.current_player == loser_player:
i.reward = reward
else:
i.reward = -reward
if counter_step > max_turn_episode:
# Terminate episode, set reward to 0
reward = 0
for i in list_training:
i.reward = reward
terminal = True
print("-----")
return list_training
def expand_node(self, model_deep_net, player_color, label_encoder,
epsilon=AlphaZeroConfig.MCTS_EPSILON,
alpha_diri=AlphaZeroConfig.MCTS_ALPHA_DIRICHLET,
cpuct=AlphaZeroConfig.MCTS_PUCT,
greed_attack = False):
"""
This function contains 2 steps on the MCTS.
Select and Expand & Evaluate
:param model_deep_net: The neural network model
:param player_color: why did I include this???
:param label_encoder: The encoder used to encode the action key
:param epsilon: hyperparameter for using the dirichlet random proba
:param alpha_diri: hyperparameter of dirichlet
:param cpuct: hyperparameter of the MCTS in alpha zero
:param greed_attack: HACK! the agent will prioritize attacking and promoting
:return:
"""
terminal = AIElements.is_over(self.stacked_state.head)
self.is_terminal = terminal
if not terminal:
possible_action = AIElements.get_possible_action(self.stacked_state.head)
possible_action_keys = list(possible_action.keys())
if self.p_state is None:
"""
Expand and Evaluate goes here!
"""
if self.stacked_state.head.get_player_turn() == self.maximizer:
state_stack_representation = np.array([self.stacked_state.get_deep_representation_stack()])
else:
state_stack_representation = mirror_stacked_state(self.stacked_state)
state_stack_representation = np.array([state_stack_representation.get_deep_representation_stack()])
self.p_state, self.v = model_deep_net.predict(state_stack_representation)
self.v_ = self.v[0][0]
self.v = self.v[0][0]
self.p_state = self.p_state[0]
if self.stacked_state.head.get_player_turn() != self.maximizer:
self.p_state = label_encoder.array_mirrored(self.p_state)
possible_action_ohe = label_encoder.transform(possible_action_keys).sum(axis=0)
self.p_state *= possible_action_ohe
sum_policy_state = np.sum(self.p_state)
if sum_policy_state > 0:
## normalize to sum 1
self.p_state /= sum_policy_state
else:
print("All valid moves were masked, do workaround.")
self.p_state += possible_action_ohe
self.p_state /= np.sum(self.p_state)
# Initialize num and q
for action in possible_action_keys:
self.num_state_action[action] = 0
self.q_state_action[action] = 0
next_state = AIElements.result_function(self.stacked_state.head, possible_action[action])
new_stacked_state = deepcopy(self.stacked_state)
new_stacked_state.append(next_state)
if action not in self.edge_action:
self.edge_action[action] = NodeMCTS(new_stacked_state, parent=self, root=False)
else:
"""
Select goes here
"""
best_action = ""
best_upper_confidence = -float('inf')
dirchlet_prob = np.random.dirichlet([alpha_diri] * len(possible_action_keys))
counter_loop = 0
# Randomize possible_action_keys
random.shuffle(possible_action_keys)
for action in possible_action_keys:
# Get the index of the action
index_action = label_encoder.le.transform([action])[0]
q_state_action_val = 0
num_state_action_val = 0
if action in self.q_state_action and action in self.num_state_action:
q_state_action_val = self.q_state_action[action]
num_state_action_val = self.num_state_action[action]
if self.root:
upper_confidence = q_state_action_val + \
cpuct * ((1 - epsilon) * self.p_state[index_action] + epsilon * dirchlet_prob[
counter_loop]) * \
np.sqrt(self.num_state) / (1 + num_state_action_val)
else:
upper_confidence = q_state_action_val + \
cpuct * self.p_state[index_action] * \
np.sqrt(self.num_state) / (1 + num_state_action_val)
if greed_attack and possible_action[action]['action'] == 'attack':
upper_confidence += AlphaZeroConfig.Q_ATTACK_GREEDY # Higher Chance to Attack
if greed_attack and possible_action[action]['action'] == 'promote':
upper_confidence += AlphaZeroConfig.Q_PROMOTE_GREEDY # Higher Chance to promote
counter_loop += 1
if best_upper_confidence < upper_confidence:
best_upper_confidence = upper_confidence
best_action = action
# Expand the node and check if this node is terminal
next_state = AIElements.result_function(self.stacked_state.head, possible_action[action])
new_stacked_state = deepcopy(self.stacked_state)
new_stacked_state.append(next_state)
if action not in self.edge_action:
self.edge_action[action] = NodeMCTS(new_stacked_state, parent=self, root=False)
self.selected_action = best_action
else:
self.v = self.stacked_state.head.sparse_eval(self.stacked_state.head.get_player_turn())