def _learn(max_episode, epsilon, alpha, save_file): """Learn by episodes. Make two TD agent, and repeat self play for given episode count. Update state values as reward coming from the environment. Args: max_episode (int): Episode count. epsilon (float): Probability of exploration. alpha (float): Step size. save_file: File name to save result. """ reset_state_values() env = TicTacToeEnv() agents = [TDAgent('O', epsilon, alpha), TDAgent('X', epsilon, alpha)] start_mark = 'O' for i in tqdm(range(max_episode)): episode = i + 1 env.show_episode(False, episode) # reset agent for new episode for agent in agents: agent.episode_rate = episode / float(max_episode) env.set_start_mark(start_mark) state = env.reset() _, mark = state done = False while not done: agent = agent_by_mark(agents, mark) ava_actions = env.available_actions() env.show_turn(False, mark) action = agent.act(state, ava_actions) # update (no rendering) nstate, reward, done, info = env.step(action) print((state,reward,action)) agent.backup(state, nstate, reward) if done: env.show_result(False, mark, reward) # set terminal state value set_state_value(state, reward) _, mark = state = nstate # rotate start start_mark = next_mark(start_mark) # save states save_model(save_file, max_episode, epsilon, alpha)
def train_agents(opponent, max_episode, epsilon, epsilon_decay, alpha, alpha_decay, gamma, render=False): reset_state_values() env = TicTacToeEnv() if opponent == 'random': agents = [ QAgent(env.observation_space.n, env.action_space.n, 'O', epsilon, epsilon_decay, alpha, alpha_decay, gamma), RandomAgent('X') ] else: # Two Q agents agents = [ QAgent(env.observation_space.n, env.action_space.n, 'O', epsilon, epsilon_decay, alpha, alpha_decay, gamma), QAgent(env.observation_space.n, env.action_space.n, 'X', epsilon, epsilon_decay, alpha, alpha_decay, gamma) ] start_mark = 'O' agent_rewards = {'O': [], 'X': []} episode = 0 for i in tqdm(range(max_episode)): episode += 1 env.show_episode(False, episode) # reset agent for new episode for agent in agents: agent.episode_rate = episode / float(max_episode) env.set_start_mark(start_mark) state = env.reset() s, mark = state done = False while not done: if render: env.render() agent = agent_by_mark(agents, mark) ava_actions = env.available_actions() env.show_turn(False, mark) action = agent.act(state, ava_actions) # update (no rendering) nstate, reward, done, info = env.step(action) agent.update(s, nstate[0], action, reward, done) if done: if render: env.render() env.show_result(render, mark, reward) # set terminal state value set_state_value(state, reward) agent_rewards['O'].append(reward) agent_rewards['X'].append(-reward) s, mark = state = nstate # rotate start start_mark = next_mark(start_mark) return agent_rewards, agent_by_mark(agents, 'O')