def _learn(max_episode, epsilon, alpha, save_file):
"""Learn by episodes.
Make two TD agent, and repeat self play for given episode count.
Update state values as reward coming from the environment.
Args:
max_episode (int): Episode count.
epsilon (float): Probability of exploration.
alpha (float): Step size.
save_file: File name to save result.
"""
reset_state_values()
env = TicTacToeEnv()
agents = [TDAgent('O', epsilon, alpha),
TDAgent('X', epsilon, alpha)]
start_mark = 'O'
for i in tqdm(range(max_episode)):
episode = i + 1
env.show_episode(False, episode)
# reset agent for new episode
for agent in agents:
agent.episode_rate = episode / float(max_episode)
env.set_start_mark(start_mark)
state = env.reset()
_, mark = state
done = False
while not done:
agent = agent_by_mark(agents, mark)
ava_actions = env.available_actions()
env.show_turn(False, mark)
action = agent.act(state, ava_actions)
# update (no rendering)
nstate, reward, done, info = env.step(action)
print((state,reward,action))
agent.backup(state, nstate, reward)
if done:
env.show_result(False, mark, reward)
# set terminal state value
set_state_value(state, reward)
_, mark = state = nstate
# rotate start
start_mark = next_mark(start_mark)
# save states
save_model(save_file, max_episode, epsilon, alpha)
def train_agents(opponent,
max_episode,
epsilon,
epsilon_decay,
alpha,
alpha_decay,
gamma,
render=False):
reset_state_values()
env = TicTacToeEnv()
if opponent == 'random':
agents = [
QAgent(env.observation_space.n, env.action_space.n, 'O', epsilon,
epsilon_decay, alpha, alpha_decay, gamma),
RandomAgent('X')
]
else: # Two Q agents
agents = [
QAgent(env.observation_space.n, env.action_space.n, 'O', epsilon,
epsilon_decay, alpha, alpha_decay, gamma),
QAgent(env.observation_space.n, env.action_space.n, 'X', epsilon,
epsilon_decay, alpha, alpha_decay, gamma)
]
start_mark = 'O'
agent_rewards = {'O': [], 'X': []}
episode = 0
for i in tqdm(range(max_episode)):
episode += 1
env.show_episode(False, episode)
# reset agent for new episode
for agent in agents:
agent.episode_rate = episode / float(max_episode)
env.set_start_mark(start_mark)
state = env.reset()
s, mark = state
done = False
while not done:
if render:
env.render()
agent = agent_by_mark(agents, mark)
ava_actions = env.available_actions()
env.show_turn(False, mark)
action = agent.act(state, ava_actions)
# update (no rendering)
nstate, reward, done, info = env.step(action)
agent.update(s, nstate[0], action, reward, done)
if done:
if render:
env.render()
env.show_result(render, mark, reward)
# set terminal state value
set_state_value(state, reward)
agent_rewards['O'].append(reward)
agent_rewards['X'].append(-reward)
s, mark = state = nstate
# rotate start
start_mark = next_mark(start_mark)
return agent_rewards, agent_by_mark(agents, 'O')