def test(self, episodes=100):
player, opp = (TfAgent(self), RandomAgent())
players = (player, opp)
winners = {pl: 0 for pl in players}
marses = {pl: 0 for pl in players}
kokses = {pl: 0 for pl in players}
for episode in tqdm(range(episodes)):
game = bg.Game(players=players, show_logs=False)
winner, status = game.play()
winners[winner] += 1
if status == 2:
marses[winner] += 1
elif status == 3:
kokses[winner] += 1
winners_total = sum(list(winners.values()))
tqdm.write(
f"[Episode {episode}] {player} vs {opp} "
f"{winners[player]}:{winners[opp]} of {winners_total} games "
f"- ({(winners[player] / winners_total) * 100.0:.2f})% "
f"| Mars-{marses[player]}/{marses[opp]}; Koks-{kokses[player]}/{kokses[opp]}"
)
players = tuple(reversed(players))
def train(
self, episodes: int,
validation: int = None,
save_path: PathLike = None,
gamma: float = 1.0, epsilon: float = 0.2,
) -> None:
"""
:param episodes:
:param validation:
:param save_path: path to save models and logs.
:param gamma: γ - discount factor. Is used to balance immediate and future reward.
:param epsilon: ε - chance to get random move in ε-greedy policy
"""
if save_path:
save_path = pathlib.Path(save_path) / f'SIGMOID_24_negative_reward_gamma_{gamma}_epsilon_{epsilon}_q_learning'
save_path.mkdir(exist_ok=False, parents=False)
for episode in tqdm(range(episodes)):
if validation is not None and not (episode + 1) % validation:
self.validate(path=save_path)
if save_path:
self.save(save_path, episode + 1)
self.model.train()
players = (self.agent_cls(self.model), self.agent_cls(self.model))
game = bg.Game(players=players)
with self.e_greedy_get_action(episode, epsilon=epsilon):
for agent, new_board, prev_board, move, available_moves in game.play_step_by_step():
agent: agents.NNAgent
pred_q = agent.estimate(board=prev_board)
if new_board.status:
reward = new_board.status
self.update(pred_q, torch.Tensor([reward]))
with prev_board.reverse() as reversed_board:
self.update(agent.estimate(board=reversed_board), torch.Tensor([-reward]))
break
else:
estimated_moves = list(agent.estimate_moves(available_moves=available_moves, board=prev_board))
agent_checkers, opp_checkers = prev_board.to_schema()
if estimated_moves:
max_q = np.max(estimated_moves)
new_q = gamma * max_q
else:
# it is too bad, if we could not make any step.
new_q = torch.Tensor([-1])
self.update(pred_q, new_q)
def validate(self, episodes: int = 100, path: pathlib.Path = None) -> None:
"""Compare current model against random player, save it on disk.
:param episodes: number of games to play against random player.
:param path: if specified, log into it.
"""
@contextmanager
def validate_log_open(path: pathlib.Path = None):
"""Open validate file or write into stdout, if it is not specified."""
if path:
path = path / 'validate.log'
path.touch(exist_ok=True)
with open(path, 'a') as f:
yield f
else:
yield sys.stdout
self.model.eval()
player, opp = self.agent_cls(self.model), agents.RandomAgent()
players = (player, opp)
winners = {pl: 0 for pl in players}
marses = {pl: 0 for pl in players}
kokses = {pl: 0 for pl in players}
with validate_log_open(path) as f:
for episode in tqdm(range(episodes)):
game = bg.Game(players=players, show_logs=False)
winner, status = game.play()
winners[winner] += 1
if status == 2:
marses[winner] += 1
elif status == 3:
kokses[winner] += 1
winners_total = sum(list(winners.values()))
tqdm.write(
f"[Episode {episode}] {player} vs {opp} "
f"{winners[player]}:{winners[opp]} of {winners_total} games "
f"- ({(winners[player] / winners_total) * 100.0:.2f})% "
f"| Mars-{marses[player]}/{marses[opp]}; Koks-{kokses[player]}/{kokses[opp]}",
file=f
)
players = tuple(reversed(players))
tqdm.write('____________________________________________________________', file=f)
def check():
model = torch.nn.Sequential(
torch.nn.Linear(24, 40),
# torch.nn.Linear(720, 100),
torch.nn.ReLU(),
torch.nn.Linear(40, 1)
).cuda()
model.load_state_dict(
torch.load(pathlib.Path('./models/24_negative_reward_gamma_0.9_epsilon_0.3_q_learning/1500.pth')))
model.eval()
player, opp = agents.NNAgent(model), agents.RandomAgent()
players = (player, opp)
game = bg.Game(players=players, show_logs=True)
game.board = bg.Board.from_schema( {22: 11, }, {23: 5, 22: 4})
available_moves = game.board.get_available_moves((1, 1))
result = list(player.estimate_moves(available_moves, game.board))
result
def play(episodes=100):
# model = torch.nn.Sequential(
# torch.nn.Linear(24, 100),
# # torch.nn.Linear(720, 100),
# torch.nn.ReLU(),
# torch.nn.Linear(100, 1)
# ).cuda()
# model.load_state_dict(torch.load(pathlib.Path('./models/720_gamma_0.95_epsilon_0.3_q_learning/1500.pth')))
# model.eval()
#
# player, opp = agents.NNAgent(model), agents.RandomAgent()
player, opp = agents.RandomAgent(), agents.RandomAgent()
players = (player, opp)
winners = {pl: 0 for pl in players}
marses = {pl: 0 for pl in players}
kokses = {pl: 0 for pl in players}
for episode in tqdm(range(episodes)):
game = bg.Game(players=players, show_logs=True)
winner, status = game.play()
winners[winner] += 1
if status == 2:
marses[winner] += 1
elif status == 3:
kokses[winner] += 1
winners_total = sum(list(winners.values()))
tqdm.write(
f"[Episode {episode}] {player} vs {opp} "
f"{winners[player]}:{winners[opp]} of {winners_total} games "
f"- ({(winners[player] / winners_total) * 100.0:.2f})% "
f"| Mars-{marses[player]}/{marses[opp]}; Koks-{kokses[player]}/{kokses[opp]}",
file=None
)
player, opp = [
agents.TCPAgent.with_server(initializer(path, model_cls), port=port)
for port, path, model_cls in zip(ports, PATHS, (Model, OtherModel))
]
players = (player, opp)
time.sleep(1)
winners = {pl: 0 for pl in players}
marses = {pl: 0 for pl in players}
kokses = {pl: 0 for pl in players}
for episode in tqdm(range(100)):
game = bg.Game(players=players, show_logs=False)
winner, status = game.play()
winners[winner] += 1
if status == 2:
marses[winner] += 1
elif status == 3:
kokses[winner] += 1
winners_total = sum(list(winners.values()))
tqdm.write(
f"[Episode {episode}] {player} vs {opp} "
f"{winners[player]}:{winners[opp]} of {winners_total} games "
f"- ({(winners[player] / winners_total) * 100.0:.2f})% "
def train(self):
tf.train.write_graph(self.sess.graph,
self.path,
'model.pb',
as_text=False)
summary_writer = tf.summary.FileWriter(
os.path.join(self.summaries_path, str(int(time.time()))),
self.sess.graph)
# the agent plays against itself, making the best move for each player
players = (player, opp) = (TfAgent(self), TfAgent(self))
validation_interval = 500
episodes = 3000
for episode in tqdm(range(episodes)):
game = bg.Game(players=players)
game_step = 0
x = self.extract_features(game.board)
while game.board.status is None:
current_player = next(game.players_steps)
with game.board.reverse(
fake=(current_player == player)) as board:
game.make_step(player=current_player, board=board)
x_next = self.extract_features(game.board)
V_next = self.get_output(x_next)
self.sess.run(self.train_op,
feed_dict={
self.x: x,
self.V_next: V_next
})
x = x_next
game_step += 1
V_next = 0 if current_player != player else 1
_, global_step, summaries, _ = self.sess.run([
self.train_op, self.global_step, self.summaries_op,
self.reset_op
],
feed_dict={
self.x:
x,
self.V_next:
np.array(
[[V_next]],
dtype='float')
})
summary_writer.add_summary(summaries, global_step=global_step)
winner = 'X' if current_player == players[0] else 'O'
# print(f"Game {episode}/{episodes} (Winner: {winner}) in {game_step} turns")
if (episode + 1) % validation_interval == 0:
self.saver.save(self.sess,
os.path.join(self.checkpoint_path,
'checkpoint'),
global_step=global_step)
self.test(episodes=100)
summary_writer.close()
self.test(episodes=1000)
"""
all_dice = (dice for dice in store['dice'])
all_moves = (moves for moves in store['moves'])
class FakeRandomAgent(RandomAgent):
def get_action(self, available_moves: Set[bg.Moves],
board: bg.Board) -> bg.Moves:
return next(all_moves)
def roll_dice(*args, **kwargs):
return next(all_dice)
if __name__ == '__main__':
t1 = time.time()
players = (FakeRandomAgent(), FakeRandomAgent())
game = bg.Game(players=players,
show_logs=False,
who_start=store['who_start'])
bg.roll_dice = roll_dice
status = game.play()
print(game.board)
print(status)
print(game._store)
print(time.time() - t1)