def result(state, action): copy_state = TicTacToe(3) copy_state.initialize_game_with_another_game(state) copy_state.make_move(action) return copy_state
class TicTacToeTDQLearningAgent: """This class represents an TD-Q learning Agent. The Agent will learn a strategy for 9 or 16 Field Tic Tac Toe. A learned strategy is stored in an database. Before the agent can suggest actions for Tic Tac Toe he needs to learn in multiple training games.""" def __init__(self, TicTacToeDBName, ticTacToeDimension): logging.basicConfig(filename=str(TicTacToeDBName).replace(".db", "Log"), filemode='a', format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s', datefmt='%H:%M:%S', level=logging.DEBUG) self.__DBCursor = None self.__DBConnection = None self.createDB(TicTacToeDBName) self.__s = None self.__a = None self.__r = None self.__dimension = ticTacToeDimension self.__random_factor = 0 self.__zobristHash = TicTacToeZobrist() if self.__dimension == 3: self.__zobristHash.set_zobrist_board_positoin_array( TicTacToeZobrist().getFixed9FieldTicTacToeZobristBoardPositionArray()) elif self.__dimension == 4: self.__zobristHash.set_zobrist_board_positoin_array( TicTacToeZobrist().getFixed16FieldTicTacToeZobristBoardPositionArray()) def explorationStrategy(self, ticTacToeState): """Function will define an exploration strategy, which is needed to learn a better strategy with TD-Q-Learn. The exploration will be more often random when the learning just started and the more training games where played the more less random exploration will be processed. Parameters ---------- TticTacToeState: TicTacToe An instance of class TicTacToe - will represent an exact board position of the game. Returns ------- tuple Returns either a random, learned action given a game state or None for terminal game states.""" if not ticTacToeState.is_terminal(): depth = ticTacToeState.count_of_game_tokens_in_game() if randint(0, self.__random_factor * depth) == randint(0, self.__random_factor * depth) and depth < (2 * self.__dimension): moves = ticTacToeState.get_possible_moves() return moves[randint(0, (len(moves) - 1))] else: return self.suggestAction(ticTacToeState) else: return None def learnTictactoe(self, gamesToPlay): """Represents the self play and learning mode of the TD-Q-Agent. Parameters ---------- gamesToPlay : int The amount of training games to play against itself.""" for gameCount in range(gamesToPlay): ttt = TicTacToe(self.__dimension) logging.info('Learning against itself game: ' + str(gameCount)) while not ttt.is_terminal(): suggestedAction = self.qLearnIteration(ttt, ttt.getReward(), 0.4, 1) ttt.make_move(suggestedAction) if ttt.is_terminal(): self.qLearnIteration(ttt, ttt.getReward(), 0.4, 1) self.__s = None self.__a = None self.__r = None if gameCount % 100 == 0: self.__random_factor += 1 def suggestAction(self, state): """Will suggest an action in a state from a learned strategy. Parameters ---------- state : TicTacToe A Tic Tic Tac Toe game situation.""" ticTacToeStateHash = self.__zobristHash.get_hash(state.game_matrix) self.__DBCursor.execute("SELECT * FROM Q WHERE ticTacToeStateHash = ?", [ticTacToeStateHash]) listOfQEntries = self.__DBCursor.fetchall() if not len(listOfQEntries) > 0: return state.get_possible_moves()[0] if state.get_player_to_move() == 'X': maxQEntry = max(listOfQEntries, key=itemgetter(2)) else: maxQEntry = min(listOfQEntries, key=itemgetter(2)) return self.translateActionTupleString2IntTuple(str(maxQEntry[1])) def qLearnIteration(self, sPrime, rPrime, alpha, gamma): """The function will learn and update a Q-Function. The Q-Function is a database and will represent a learned TD-Q strategy. Parameters ---------- sPrime : TicTacToe A Tic Tac Toe game situation. rPrime : float A reward for an action in a state. alpha : float The learning rate for stronger or weaker TD update of Q-Values. gamma: float The discounting factor for future rewards. Returns ------- tuple An action tuple which should be explored next.""" if sPrime.is_terminal(): self.insertActionValueInQ(sPrime, None, rPrime) if self.__s is not None: qActionValue = self.getActionValueFromQ(self.__s, self.__a) maxActionValue = self.maxActionValueForAllActionsInSPrime(sPrime) qValueUpdate = qActionValue + alpha * (self.__r + gamma * maxActionValue - qActionValue) self.insertActionValueInQ(self.__s, self.__a, qValueUpdate) logging.info('\nUpdate Q-Value:') logging.info( 'Q(' + str(self.__zobristHash.get_hash(self.__s.game_matrix)) + ', ' + str(self.__a) + ') <-- ' + str( qActionValue) + ' + ' + str(alpha) + '(' + str(self.__r) + ' + ' + str(gamma) + ' * ' + str( maxActionValue) + ' - ' + str(qActionValue) + ') = ' + str(qValueUpdate)) logging.info('\n' + sPrime.printable_game_matrix()) logging.info(self.__zobristHash.get_hash(sPrime.game_matrix)) else: self.__s = TicTacToe(self.__dimension) self.__s.initialize_game_with_another_game(sPrime) self.__a = self.explorationStrategy(sPrime) self.__r = rPrime return self.__a ########################################################################### ### AGENT EXPERIENCE (DATABASE) FUNCTIONS ### ########################################################################### def createDB(self, TicTacToeDBName): """Creates a SQLite table - represents the learned Q-Function. Parameters ---------- TicTacToeDBName : str Name of the db (learned Q-Function or strategy representation).""" self.__DBConnection = sqlite3.connect(TicTacToeDBName) self.__DBCursor = self.__DBConnection.cursor() self.__DBCursor.execute( 'CREATE TABLE IF NOT EXISTS Q(tictacToeStateHash INTEGER, actionInState TEXT, actionValue REAL)') def insertActionValueInQ(self, sPrime, actionInState, actionValue): """Inserts a given Q-Value into the Q table. Parameters ---------- sPrime : TicTacToe A Tic Tac Toe game situation. actionInState : tuple A possible action in a Tic Tac Toe state. actionValue : float A Q-Value.""" ticTacToeStateHash = self.__zobristHash.get_hash(sPrime.game_matrix) actionInState = str(actionInState) if self.isStateActionPairInQ(ticTacToeStateHash, actionInState): self.__DBCursor.execute("UPDATE Q Set actionValue = ? WHERE ticTacToeStateHash = ? AND actionInState = ?", [actionValue, ticTacToeStateHash, actionInState]) else: self.__DBCursor.execute("INSERT INTO Q(ticTacToeStateHash, actionInState, actionValue) VALUES (?, ?, ?)", (ticTacToeStateHash, actionInState, actionValue)) self.__DBConnection.commit() def getActionValueFromQ(self, sPrime, actionInState): """Returns a Q-Value with a Tic Tac Toe game situation and a possible action as index. Parameters ---------- sPrime : TicTacToe A Tic Tac Toe game situation. actionInState : tuple A possible action in a Tic Tac Toe state. Returns ------- float 0 if no result is found or a found Q-Value.""" ticTacToeStateHash = self.__zobristHash.get_hash(sPrime.game_matrix) actionInState = str(actionInState) self.__DBCursor.execute( "SELECT actionValue FROM Q WHERE ticTacToeStateHash = ? AND actionInState = ?", [ticTacToeStateHash, actionInState]) results = self.__DBCursor.fetchall() if len(results) > 0: return results[0][0] else: self.insertActionValueInQ(sPrime, actionInState, 0) return 0 def maxActionValueForAllActionsInSPrime(self, sPrime): """Returns the maximal possible learned action value (Q-Value) for a given state. Parameters ---------- sPrime : TicTacToe A Tic Tac Toe game situation. Returns ------- float Returns either the reward for the given state if no Q-Values are found or the maximal found Q-Value for a given state.""" if not sPrime.is_terminal(): allActionsInSPrime = sPrime.get_possible_moves() maxActionValue = self.getActionValueFromQ(sPrime, allActionsInSPrime[0]) for action in allActionsInSPrime: maxActionValue = max(maxActionValue, self.getActionValueFromQ(sPrime, action)) return maxActionValue else: return sPrime.getReward() def minActionValueForAllActionsInSPrime(self, sPrime): """Returns the minimal possible learned action value (Q-Value) for a given state. Parameters ---------- sPrime : TicTacToe A Tic Tac Toe game situation. Returns ------- float Returns either the reward for the given state if no Q-Values are found or the minimal found Q-Value for a given state.""" if not sPrime.is_terminal(): allActionsInSPrime = sPrime.get_possible_moves() minActionValue = self.getActionValueFromQ(sPrime, allActionsInSPrime[0]) for action in allActionsInSPrime: minActionValue = min(minActionValue, self.getActionValueFromQ(sPrime, action)) return minActionValue else: return sPrime.getReward() def isStateActionPairInQ(self, ticTacToeStateHash, actionInState): """Checks if state action pair is in the database. Parameters ---------- ticTacToeStateHash : int Is the zobrist hash value of a Tic Tac Toe game situation (state). actionInState : tuple Is an action which is possible in a given game situation. Returns ------- bool True if state action is in database, False if not.""" self.__DBCursor.execute("SELECT * FROM Q WHERE ticTacToeStateHash = ? AND actionInState = ?", [ticTacToeStateHash, actionInState]) if len(self.__DBCursor.fetchall()) == 0: return False else: return True def translateActionTupleString2IntTuple(self, stringActionTuple): """Translates a string like str((0,0)) into an integer tuple (0,0). Parameters ---------- stringActionTuple : str The action tuple in string format. Returns ------- tuple The action tuple in tuple format with int values.""" tupleStringValues = stringActionTuple.translate(None, '( )').split(',') return (int(tupleStringValues[0]), int(tupleStringValues[1])) def closeDB(self): """The funcions will close the database.""" self.__DBCursor.close() self.__DBConnection.close() ########################################################################### ### FIXED LEARNING FUNCTIONS ### ########################################################################### def train3x3TicTacToeAgentIn100Games(self): """Function will learn a 9 Field Tic Tac Toe strategy (Q-Function) in 100 training games. The strategy will be stores as database.""" agent = TicTacToeTDQLearningAgent(TICTACTOE_3x3_TDQ_AGENT_100_NAME, 3) agent.learnTictactoe(100) agent.closeDB() def train3x3TicTacToeAgentIn1000Games(self): """Function will learn a 9 Field Tic Tac Toe strategy (Q-Function) in 1.000 training games. The strategy will be stores as database.""" agent = TicTacToeTDQLearningAgent(TICTACTOE_3x3_TDQ_AGENT_1000_NAME, 3) agent.learnTictactoe(1000) agent.closeDB() def train3x3TicTacToeAgentIn10000Games(self): """Function will learn a 9 Field Tic Tac Toe strategy (Q-Function) in 10.000 training games. The strategy will be stores as database.""" agent = TicTacToeTDQLearningAgent(TICTACTOE_3x3_TDQ_AGENT_10000_NAME, 3) agent.learnTictactoe(10000) agent.closeDB() def train4x4TicTacToeAgentIn100Games(self): """Function will learn a 16 Field Tic Tac Toe strategy (Q-Function) in 100 training games. The strategy will be stores as database.""" agent = TicTacToeTDQLearningAgent(TICTACTOE_4x4_TDQ_AGENT_100_NAME, 4) agent.learnTictactoe(100) agent.closeDB() def train4x4TicTacToeAgentIn1000Games(self): """Function will learn a 16 Field Tic Tac Toe strategy (Q-Function) in 1.000 training games. The strategy will be stores as database.""" agent = TicTacToeTDQLearningAgent(TICTACTOE_4x4_TDQ_AGENT_1000_NAME, 4) agent.learnTictactoe(1000) agent.closeDB() def train4x4TicTacToeAgentIn10000Games(self): """Function will learn a 16 Field Tic Tac Toe strategy (Q-Function) in 10.000 training games. The strategy will be stores as database.""" agent = TicTacToeTDQLearningAgent(TICTACTOE_4x4_TDQ_AGENT_10000_NAME, 4) agent.learnTictactoe(10000) agent.closeDB()