def _monte_carlo_sample(game_spec, board_state, side, policy = False, session = None, input_layer = None, output_layer = None, valid_only = False, cnn_on = False): """Sample a single rollout from the current board_state and side. Moves are made to the current board_state until we reach a terminal state then the result and the first move made to get there is returned. Args: game_spec (BaseGameSpec): The specification for the game we are evaluating board_state (3x3 tuple of int): state of the board side (int): side currently to play. +1 for the plus player, -1 for the minus player Returns: (result(int), move(int,int)): The result from this rollout, +1 for a win for the plus player -1 for a win for the minus player, 0 for a draw """ result = game_spec.has_winner(board_state) if result != None: return result, None moves = list(game_spec.available_moves(board_state)) if not moves: return 0, None # select a random move if policy: # get stochastic network move gives wrong type (array of 81 elements instead of tuple), so we need to reconfigure move = get_stochastic_network_move(session, input_layer, output_layer, board_state, side, valid_only, game_spec, cnn_on) move = game_spec.flat_move_to_tuple([i for i,x in enumerate(move) if x == 1][0]) else: move = random.choice(moves) result, next_move = _monte_carlo_sample(game_spec, game_spec.apply_move(board_state, move, side), -side, policy, session, input_layer, output_layer, valid_only, cnn_on) return result, move
def make_move_historical(histoical_network_index, board_state, side): net = historical_networks[histoical_network_index] move = get_stochastic_network_move(session, net[0], net[1], board_state, side, valid_only=True, game_spec=game_spec) return game_spec.flat_move_to_tuple(move.argmax())
def make_training_move(board_state, side): mini_batch_board_states.append(np.ravel(board_state) * side) move = get_stochastic_network_move(session, input_layer, output_layer, board_state, side, valid_only=True, game_spec=game_spec) mini_batch_moves.append(move) return game_spec.flat_move_to_tuple(move.argmax())
def network_player(board_state, side): print print "Network player (%s)" % side tic_tac_toe.print_game_state(board_state) move_probs = network_helpers.get_stochastic_network_move( session, input_layer, output_layer, board_state, side, log=True) move = game_spec.flat_move_to_tuple(move_probs.argmax()) print "Network move:", move return move
def make_training_move(board_state, side): if cnn_on: # We must have the first 3x3 board as first 9 entries of the list, second 3x3 board as next 9 entries etc. # This is required for the CNN. The CNN takes the first 9 entries and forms a 3x3 board etc. """If the 10 split 3x3 boards are desired, use create_3x3_board_states(board_state) here""" np_board_state = create_3x3_board_states(board_state) else: np_board_state = np.array(board_state) np_board_state[np_board_state > 1] = 0 mini_batch_board_states.append( np_board_state * side ) # append all states are used in the minibatch (+ and - determine which player's state it was) rand_numb = random.uniform(0., 1.) if rand_numb < eps: move = get_random_network_move(board_state, game_spec) elif deterministic: move = get_deterministic_network_move(session, input_layer, output_layer, board_state, side, valid_only=True, game_spec=game_spec, cnn_on=cnn_on) else: if mcts: _, move = monte_carlo_tree_search(game_spec, board_state, side, 27, session, input_layer, output_layer, True, cnn_on, True) else: move = get_stochastic_network_move(session, input_layer, output_layer, board_state, side, valid_only=True, game_spec=game_spec, cnn_on=cnn_on) move_for_game = np.asarray( move ) # The move returned to the game is in a different configuration than the CNN learn move if cnn_on: # Since the mini batch states is saved the same way it should enter the neural net (the adapted board state), # the same should happen for the mini batch moves move = create_3x3_board_states(np.reshape( move, [9, 9])) # The function requires a 9x9 array mini_batch_moves.append(move[0:81]) else: mini_batch_moves.append(move) return game_spec.flat_move_to_tuple(move_for_game.argmax())
def make_training_move(board_state, side): if cnn_on: np_board_state = create_3x3_board_states(board_state) else: np_board_state = np.array(board_state) mini_batch_board_states.append(np_board_state * side) rand_numb = random.uniform(0., 1.) if rand_numb < eps: move = get_random_network_move(board_state, game_spec) elif deterministic: move = get_deterministic_network_move(session, input_layer, output_layer, board_state, side, valid_only=True, game_spec=game_spec, cnn_on=cnn_on) else: if mcts: _, move = monte_carlo_tree_search(game_spec, board_state, side, 27, session, input_layer, output_layer, True, cnn_on, True) else: move = get_stochastic_network_move(session, input_layer, output_layer, board_state, side, valid_only=True, game_spec=game_spec, cnn_on=cnn_on) move_for_game = np.asarray( move ) # The move returned to the game is in a different configuration than the CNN learn move if cnn_on: # Since the mini batch states is saved the same way it should enter the neural net (the adapted board state), # the same should happen for the mini batch moves move = create_3x3_board_states(np.reshape( move, [9, 9])) # The function requires a 9x9 array mini_batch_moves.append(move[0:81]) else: mini_batch_moves.append(move) return game_spec.flat_move_to_tuple(move_for_game.argmax())
def make_move_historical(net, board_state, side): if mcts: _, move = monte_carlo_tree_search(game_spec, board_state, side, 27, session, input_layer, output_layer, True, cnn_on, True) else: # move = get_deterministic_network_move(session, net[0], net[1], board_state, side, # valid_only = True, game_spec = game_spec, cnn_on = cnn_on) move = get_stochastic_network_move(session, net[0], net[1], board_state, side, valid_only=True, game_spec=game_spec, cnn_on=cnn_on) move_for_game = np.asarray( move) # move must be an array, mcts doesn't return this return game_spec.flat_move_to_tuple(move_for_game.argmax())
def predict_best_move_low_level(game_spec, create_network, network_file_path, player, board_state): """Make a predicition for the next move at a given state using some lower level parameters Args: create_network (->(input_layer : tf.placeholder, output_layer : tf.placeholder, variables : [tf.Variable])): Method that creates the network we will train. network_file_path (str): path to the file with weights we want to load for this network game_spec (games.base_game_spec.BaseGameSpec): The game we are playing player: The player to make the move 1 or -1 board_state: The state of the board at some time during the game Returns: a vector of zeros with a 1 on the position which represents the best move to be taken """ reward_placeholder = tf.placeholder("float", shape=(None, )) actual_move_placeholder = tf.placeholder("float", shape=(None, game_spec.outputs())) input_layer, output_layer, variables = create_network() policy_gradient = tf.log( tf.reduce_sum(tf.mul(actual_move_placeholder, output_layer), reduction_indices=1)) * reward_placeholder with tf.Session() as session: session.run(tf.initialize_all_variables()) if network_file_path and os.path.isfile(network_file_path): print("Loading trained network from ", network_file_path) load_network(session, variables, network_file_path) else: print("File with trained network can't be loaded. Exiting...'") return return get_stochastic_network_move(session, input_layer, output_layer, board_state, player)
def make_training_move(board_state, side): mini_batch_board_states.append(np.ravel(board_state) * side) move = get_stochastic_network_move(session, input_layer, output_layer, board_state, side) mini_batch_moves.append(move) #print("Training Move Called") return game_spec.flat_move_to_tuple(move.argmax())
import collections