def generate_move_random(board: np.ndarray, player: BoardPiece, saved_state: Optional[SavedState], args=None) \ -> Tuple[PlayerAction, SavedState]: """ :param board: State of board, 6 x 7 with either 0 or player ID [1, 2] :param player: Player ID of random agent :param saved_state: Not used in this implementation of the random move generation :param args: Optional parameter :return: Column in which player wants to make his move (chosen randomly) """ # Get column indexes where there is no player and choose one empty column randomly action = np.random.choice(np.unique(np.where(board == NO_PLAYER)[1])) return PlayerAction(action), SavedState()
def test_apply_player_action_fail(): """Test that an error is raised if an action in a full or not existent column is applied""" # Test for the insertion in a already full column full_board = initialize_game_state() full_board[:] = PLAYER1 # Fill the board completely with one player n_cols = full_board.shape[1] for i in range( n_cols): # Check that the exception is raised in every column with pytest.raises(Exception) as e: assert apply_player_action(full_board, PlayerAction(i), PLAYER1) assert str( e.value ) == "Tried to apply an action in a non existent or full column" # Test for a non existent column with pytest.raises(Exception) as e: assert apply_player_action(initialize_game_state(), PlayerAction(100), PLAYER1) assert str( e.value ) == "Tried to apply an action in a non existent or full column"
def test_random(): from agents.agents_random.random import generate_move_random board = np.array([[1, 2, 2, 0, 1, 2, 2], [2, 1, 1, 2, 1, 2, 2], [2, 2, 1, 1, 1, 2, 2], [2, 1, 2, 2, 2, 1, 1], [1, 2, 1, 1, 1, 2, 2], [1, 1, 2, 1, 2, 1, 2]]) action, saved_state = generate_move_random(board,BoardPiece(1),saved_state=0) assert isinstance(action,PlayerAction) assert action == PlayerAction(3) #Taking the empty one
def generate_move_random( board: np.ndarray, _player: BoardPiece, saved_state: Optional[SavedState] = None ) -> Tuple[PlayerAction, SavedState]: # Choose a valid, non-full column randomly and return it as `action` valid_columns = np.where(board[-1, :] == 0) action = PlayerAction( np.random.choice(np.array(valid_columns).flatten(), 1)) return action, saved_state
def generate_move_MCTS(board: np.ndarray, player: BoardPiece, saved_state: Optional[SavedState], max_time: float = 5) \ -> Tuple[PlayerAction, SavedState]: """ :param board: State of board, 6 x 7 with either 0 or player ID [1, 2] :param player: Player ID :param saved_state: Not used in this implementation of the move generation :param max_time: Time ins sec given to the MCTS agent to find teh next action :return: Column in which player wants to make his move (chosen using MCTS) """ # Give time sec to the agent to find a good action action = MCTS(board, player, max_time) return PlayerAction(action), SavedState()
def generate_move_minimax( board: np.ndarray, _player: BoardPiece, saved_state: Optional[SavedState]) -> Tuple[PlayerAction, SavedState]: # Choose a valid, non-full column randomly and return it as `action` depth = 4 alpha = -math.inf beta = math.inf maximizingPlayer = True action = alpha_beta(board, _player, depth, alpha, beta, maximizingPlayer)[0] return PlayerAction(action), saved_state
def generate_move_minimax(board: np.ndarray, player: BoardPiece, saved_state: Optional[SavedState], depth: int = 4) -> Tuple[PlayerAction, SavedState]: """ :param board: State of board, 6 x 7 with either 0 or player ID [1, 2] :param player: Player ID :param saved_state: Not used in this implementation of the minimax move generation :param depth: Depth of the minimax agent / how many steps should be searched ahead :return: Column in which player wants to make his move (chosen using the minimax algorithm) """ # If the minimax agent can make the first move, make sure it is always in the middle (position 3) if not board.any(): return PlayerAction(3), SavedState() # Create a list that holds the player first, and the opponent second players = [PLAYER1, PLAYER2] players.remove(player) ordered_players = [player] + players # Determine the best action using a minimax algorithm with alpha-bet-pruning which looks 4 steps ahead # (two for each player) _, action = minimax(board, -np.inf, np.inf, ordered_players, depth, True) return PlayerAction(action), SavedState()
def buildGameStateFromID(self): """ plays 4 moves on self.board if they're possible """ # positionID = eg 4256 -> 1. Move: 4th Column, 2. Move: 2nd column player = self.player for col in self.positionID: action = PlayerAction(int(col)) if move_is_possible(self.board, action): apply_player_action(self.board, action, player) player = other_player(player) else: self.status = False break
def test_apply_player_action(): from agents.common import apply_player_action, initialize_game_state board = initialize_game_state() board[5, 0] = PLAYER2 board[5, 1] = PLAYER1 board[5, 2] = PLAYER2 board[5, 3] = PLAYER1 board[5, 4] = PLAYER1 board[5, 5] = PLAYER1 copy_board = board.copy() old_board, position = apply_player_action(board, PlayerAction(3), PLAYER1, True, True) assert old_board.all() == copy_board.all() assert position == (4, 3) assert board[position] == PLAYER1 board[:, 0] = PLAYER1 position2 = apply_player_action(board, PlayerAction(0), PLAYER1, False, True) assert position2 == 0 # Return 0 if full column.
def generate_move_MCTS(board: np.ndarray, player: BoardPiece, saved_state: Optional[SavedState]) \ -> object: global PLAYER global OPPONENT PLAYER = player if PLAYER == PLAYER1: OPPONENT = PLAYER2 else: OPPONENT = PLAYER1 action = MCTS(board) return PlayerAction(action), SavedState()
def traverse(self): """ Searches the tree until a node with unexpanded children is found This function is called recursively during the selection phase of MCTS. Recursion ceases once it reaches a node with unexpanded children. At this point, a new child is created from the node's list of actions, and the remainder of the game is simulated. The stats are then updated and propagated up to the root node, which made the original call. Parameters node = node selected by root node or previous select_action call """ # Check whether the current node is a terminal state if self.state == GameState.IS_WIN: if self.max_player: return True else: return False elif self.state == GameState.IS_DRAW: return -1 # If any children are unexpanded, expand them and run a simulation if len(self.children) < len(self.actions): # Select the next randomized action in the list action = PlayerAction(self.actions[len(self.children)]) # Apply the action to the current board child_bd, child_msk = apply_action_cp(self.board, self.mask, action, self.shape) # Add the new child to the node new_child = Connect4Node(child_bd, child_msk, self.shape, action, not self.max_player) # If the game does not end, continue building the tree self.add_child(new_child) # Simulate the game to completion max_win = new_child.sim_game() # Update the child's stats new_child.update_stats(max_win) # Else, continue tree traversal else: next_node_ind = self.ucb1_select() next_child = self.children[next_node_ind] max_win = next_child.traverse() # Update new child's stats based on the result of a simulation self.update_stats(max_win) return max_win
def test_apply_player_action_success(): """Test for successful application of actions""" # Test if application of action (drop of the board piece) is possible for every cell and player for player in players: board = initialize_game_state() n_rows = board.shape[0] for action, column in enumerate(board.T): for i in range(n_rows): board = apply_player_action(board=board, action=PlayerAction(action), player=player) assert isinstance(board, np.ndarray) assert board.dtype == np.int8 assert board.shape == (6, 7) assert not np.all(board == 0) assert board[n_rows - 1 - i, action] == player
def generate_move_minimax( board: np.ndarray, player: BoardPiece, saved_state: Optional[SavedState] ) -> Tuple[PlayerAction, Optional[SavedState]]: """ :param board: np.ndarray Contains current state of the board an ndarray, shape (ROWS, COLUMNS) and data type (dtype) BoardPiece :param player: BoardPiece Current player playing the game :param saved_state: Saved state of the game :return: action: PlayerAction (np.int8) The column to be played saved_state: The saved state of the game """ col_, val = minimax(4, board, player, math.inf, -math.inf, True) action = PlayerAction(int(col_)) return action, saved_state
def generate_move_random( board: np.ndarray, player: BoardPiece, saved_state: Optional[SavedState] ) -> Tuple[PlayerAction, Optional[SavedState]]: """ Choose randomly an action based on available free columns :param board: np.ndarray: current state of the board, filled with Player pieces :param player: BoardPiece: standard generate_move input... not used in this case :param saved_state: SavedState: standard generate_move input... not used in this case :return: PlayerAction: random column to play """ # Choose a valid, non-full column randomly and return it as `action` action = PlayerAction( np.random.choice(np.arange(board.shape[1])[board[-1, :] == NO_PLAYER])) return action, saved_state
def generate_move(board: np.ndarray, player: BoardPiece, saved_state: Optional[SavedState])\ -> Tuple[PlayerAction, Optional[SavedState]]: """ generates an optimal move/action using the Monte Carlo Tree Search strategy :param board: current state of board :param player: player whose move is optimized :param saved_state: saved state of board :return: move, saved_state (optional) """ # turn on RuntimeWarning np.seterr(divide='warn') global PLAYER global OPPONENT # set player and opponent PLAYER = player OPPONENT = PLAYER1 if player == PLAYER2 else PLAYER2 # if lowest board row is empty make first move in central col = 3 if not board[0, :].any(): action = 3 else: # create root Node object root = Node(board_copy=deepcopy(board), parent=None, col=-1, player=PLAYER) # create MCTS object for player mcts = MCTS(PLAYER) #to start the time # call monte carlo tree search starting from root node action = mcts.monte_carlo_tree_search(root) # return optimal action for player return PlayerAction(action), saved_state
def compute_score(self): """ uses heuristic to set score of node """ action = PlayerAction(self.positionID[3]) self.score = evaluate_heuristic(self.board, action, self.player)
def alpha_beta(board: Bitmap, mask: Bitmap, max_player: bool, depth: int, alpha: GameScore, beta: GameScore, board_shp: Tuple) -> Tuple[GameScore, Optional[PlayerAction]]: """ Recursively call alpha_beta to build a game tree to a pre-determined max depth. Once at the max depth, or at a terminal node, calculate and return the heuristic score. Scores farther down the tree are penalized. :param board: bitmap representing positions of current player :param mask: bitmap representing positions of both players :param max_player: boolean indicating whether the depth at which alpha_beta is called from is a maximizing or minimizing player :param depth: the current depth in the game tree :param alpha: the currently best score for the maximizing player along the path to root :param beta: the currently best score for the minimizing player along the path to root :param board_shp: the shape of the game board :return: the best action and the associated score """ # If the node is at the max depth or a terminal node calculate the score max_depth = 7 win_score = 150 state_p = check_end_state(board ^ mask, mask, board_shp) if state_p == GameState.IS_WIN: if max_player: return GameScore(-win_score), None else: return GameScore(win_score), None elif state_p == GameState.IS_DRAW: return 0, None elif depth == max_depth: return heuristic_solver_bits(board, mask, board_shp[0], max_player), None # For each potential action, call alpha_beta pot_actions = valid_actions(mask, board_shp) if max_player: score = -100000 action = -1 for col in pot_actions: # Apply the current action min_board, new_mask = apply_action_cp(board, mask, col, board_shp) # Call alpha-beta new_score, temp = alpha_beta(min_board, new_mask, False, depth + 1, alpha, beta, board_shp) new_score -= depth # Check whether the score updates if new_score > score: score = new_score action = col # Check whether we can prune the rest of the branch if score >= beta: # print('Pruned a branch') break # Check whether alpha updates the score if score > alpha: alpha = score return GameScore(score), PlayerAction(action) else: score = 100000 action = -1 for col in pot_actions: # Apply the current action max_board, new_mask = apply_action_cp(board, mask, col, board_shp) # Call alpha-beta new_score, temp = alpha_beta(max_board, new_mask, True, depth + 1, alpha, beta, board_shp) new_score += depth # Check whether the score updates if new_score < score: score = new_score action = col # Check whether we can prune the rest of the branch if score <= alpha: # print('Pruned a branch') break # Check whether alpha updates the score if score < beta: beta = score return GameScore(score), PlayerAction(action)
def generate_move_mcts(board: np.ndarray, player: BoardPiece, saved_state: Optional[SavedState] ) -> Tuple[PlayerAction, Optional[SavedState]]: root = MonteCarloTreeSearchNode(state=board, player=player) best_node = root.best_simulated_action() action = PlayerAction(int(best_node.parent_action)) return action, saved_state
def minimax(board: np.ndarray, alpha: int, beta: int, players: List[BoardPiece], depth: int, MaxPlayer: bool) \ -> Tuple[any, Union[PlayerAction, None]]: """ :param board: State of board, 6 x 7 with either 0 or player ID [1, 2] :param alpha: the best value that maximizer can guarantee in the current state or before in the maximizer turn :param beta: the best value that minimizer can guarantee in the current state or before it in the minimizer turn :param players: List of players with maximizer first :param depth: Steps that should be evaluated :param MaxPlayer: Bool if it is the maximizers turn :return: Best value for maximizer or minimizer and the corresponding action """ # Check endstate of the game after last players move end_state = check_end_state(board, players[0] if not MaxPlayer else players[1]) # Return very positive/negative value if the move of the last player won the game if end_state == GameState.IS_WIN: if MaxPlayer: return -10**10, None else: return 10**10, None if end_state == GameState.IS_DRAW: return 0, None # Only evaluate the board if the game is still going on and the bottom of the tree is reached if end_state == GameState.STILL_PLAYING and depth == 0: # Evaluate how good the current board is for the maximizing player return eval_board(board, players), None if MaxPlayer: best_value = -np.inf player = players[0] else: best_value = np.inf player = players[1] # Get all the possible actions (not already full columns) free_columns = np.unique(np.where(board == NO_PLAYER)[1]) # Change the order of the actions such that in case that more than one action has the same value, # a random action is selected action_values = [] for action in free_columns: # Apply the action and got one steep deep deeper into the tree board_new = apply_player_action(board.copy(), PlayerAction(action), player) value, _ = minimax(board_new, alpha, beta, players, depth - 1, not MaxPlayer) action_values.append((action, value)) # If the action results in a board that is better than all the previously checked actions # for the current player, save it and the corresponding evaluation of the board if MaxPlayer and value >= best_value: best_value = value best_action = action #alpha = max(alpha, best_value) #if beta <= alpha: # break if not MaxPlayer and value <= best_value: best_value = value best_action = action #beta = min(beta, best_value) #if beta <= alpha: #break #if depth == 4: #print(action_values) return best_value, best_action
"| |\n" + \ "| X X |\n" + \ "| O X X |\n" + \ "| O X O O |\n" + \ "| O O X X |\n" + \ "|==============|\n" + \ "|0 1 2 3 4 5 6 |" boards: List[str] = [board1, board2] # np.ndarray representations of initial states game_state1 = initialize_game_state() game_state2 = string_to_board(board2) game_states: np.ndarray = [game_state1, game_state2] # list of actions and players to test on initial states actions = [PlayerAction(0), PlayerAction(3), PlayerAction(6)] players = [PLAYER1, PLAYER2] # games state 1 with all possible player action combo game_state1_with_action1_player1 = game_state1.copy() game_state1_with_action1_player1[5][0] = PLAYER1 game_state1_with_action2_player1 = game_state1.copy() game_state1_with_action2_player1[5][3] = PLAYER1 game_state1_with_action3_player1 = game_state1.copy() game_state1_with_action3_player1[5][6] = PLAYER1 game_state1_with_action1_player2 = game_state1.copy() game_state1_with_action1_player2[5][0] = PLAYER2 game_state1_with_action2_player2 = game_state1.copy() game_state1_with_action2_player2[5][3] = PLAYER2 game_state1_with_action3_player2 = game_state1.copy() game_state1_with_action3_player2[5][6] = PLAYER2
def minimax_action(board: np.ndarray, player: BoardPiece, saved_state: Optional[SavedState]): """Minimax agent getting a board and the corresponding player turn and returning the best non-full column for the player according to the algorithm. Enter the current state of the board, and performs a top-bottom search on different positions of the board, so that the most optimal according the heuristics used is found. Args: board: Current state of the board player: Whose turn is it. saved_state: Pre-computation work Returns: action: Best column to use. saved_state_out: Tree structure """ global BOARD_VALUES tree = minmax_tree() # Weights tree initialization. other_player = None if player == BoardPiece(1): # Finding out which player is who. other_player = BoardPiece(2) elif player == BoardPiece(2): other_player = BoardPiece(1) idx1 = [] start = -1 for i in range(0, 7): # Player plays cumul1 = 0 # Initialization of the cumulative variable. old_board = board.copy() # Optimal way to start: central column. if sum(sum(old_board[:, :]) == 0) == 7: start = 10 break game, board_val = assign_weight(old_board, i, player, BOARD_VALUES) break_y, cumul1 = eval_heu(cumul1, board_val, i, idx1, game, node_type=np.array([-1])) if break_y and cumul1 > 10000: # Already a winning position, break the search. tree.child[i].value = cumul1 break elif break_y and cumul1 < 10000: # Full column, do not go down its branches. tree.child[i].value = cumul1 continue idx2 = [] for j in range(0, 7): # other player plays old_board1 = old_board.copy() game, board_val = assign_weight(old_board1, j, other_player, BOARD_VALUES) break_y, cumul2 = eval_heu(cumul1, board_val, j, idx2, game) if break_y: # Either a full-column (worst value given) or a win (best one given). tree.child[i].child[j].value = cumul2 continue idx3 = [] for k in range(0, 7): # player plays old_board2 = old_board1.copy() game, board_val = assign_weight(old_board2, k, player, BOARD_VALUES) break_y, cumul3 = eval_heu(cumul2, board_val, k, idx3, game, np.array([-1])) if break_y: # Either a full-column (worst value given) or a win (best one given). tree.child[i].child[j].child[k].value = cumul3 continue idx4 = [] for v in range(0, 7): # other player plays old_board3 = old_board2.copy() game, board_val = assign_weight(old_board3, v, other_player, BOARD_VALUES) break_y, cumul4 = eval_heu(cumul3, board_val, v, idx4, game) # Last layers' nodes assigned the top-down cumulative heuristic value. tree.child[i].child[j].child[k].child[v].value = cumul4 _, val_4 = min_child(tree.child[i].child[j].child[k], idx4) tree.child[i].child[j].child[k].value = val_4 # Assigning the value to father of minimal node. _, val_3 = max_child(tree.child[i].child[j], idx3) tree.child[i].child[j].value = val_3 # Assigning the value to father of maximal node. _, val_2 = min_child(tree.child[i], idx2) tree.child[i].value = val_2 # Assigning the value to father of minimal node. action, tree.value = max_child(tree, idx1) action = PlayerAction(action) # Action to be taken in the Class PlayerAction if start == 10: # If it is the 1st movement, 1st action performed is the optimal: column 3. action = 3 saved_state_out = tree return action, saved_state_out