def set_seeds_op(self, side: Side, hole: int, seeds: int):
if hole < 1 or hole > self.holes:
raise ValueError(
'Hole number must be between 1 and number of holes')
if seeds < 0:
raise ValueError('There has to be a non-negative number of seeds')
self.board[Side.get_index(Side.opposite(side))][self.holes + 1 -
hole] = seeds
def clone(cls, original_board):
holes = original_board.holes
board = cls(holes, 0)
for hole in range(1, holes + 1):
board.board[Side.get_index(Side.NORTH)][hole] \
= deepcopy(original_board.board[Side.get_index(Side.NORTH)][hole])
board.board[Side.get_index(Side.SOUTH)][hole] \
= deepcopy(original_board.board[Side.get_index(Side.SOUTH)][hole])
return board
def __str__(self):
board_str = str(self.board[Side.get_index(Side.NORTH)][0]) + " --"
for i in range(self.holes, 0, -1):
board_str += " " + str(self.board[Side.get_index(Side.NORTH)][i])
board_str += "\n"
for i in range(1, self.holes + 1, 1):
board_str += " " + str(self.board[Side.get_index(Side.SOUTH)][i])
board_str += " -- " + str(self.board[Side.get_index(
Side.SOUTH)][0]) + "\n"
return board_str
def __init__(self, holes: int, seeds: int):
if holes < 1:
raise ValueError('There has to be at least one hole')
if seeds < 0:
raise ValueError('There has to be a non-negative number of seeds')
self._holes = holes
# Place the seeds in the holes
self.board = [[0 for _ in range(holes + 1)] for _ in range(2)]
for hole in range(1, holes + 1):
self.board[Side.get_index(Side.NORTH)][hole] = seeds
self.board[Side.get_index(Side.SOUTH)][hole] = seeds
def _scoring_store_diff(state: MancalaEnv, parent_side: Side) -> int:
"""Calculates the differences between two stores."""
our_seeds = state.board.get_seeds_in_store(parent_side)
their_seeds = state.board.get_seeds_in_store(Side.opposite(parent_side))
reward = our_seeds - their_seeds
return reward
def add_seeds(self, side: Side, hole: int, seeds: int):
if hole < 1 or hole > self.holes:
raise ValueError(
'Hole number must be between 1 and number of holes')
if seeds < 0:
raise ValueError('There has to be a non-negative number of seeds')
self.board[Side.get_index(side)][hole] += seeds
def get_state_legal_actions(board: Board, side: Side,
north_moved: bool) -> List[Move]:
# If this is the first move of NORTH, then NORTH can use the pie rule action
legal_moves = [] if north_moved or side == side.SOUTH else [
Move(side, 0)
]
for i in range(1, board.holes + 1):
if board.board[side.get_index(side)][i] > 0:
legal_moves.append(Move(side, i))
return legal_moves
def perform_move(self, move: Move) -> int:
"""Performs a move and returns the reward for this move."""
seeds_in_store_before = self.board.get_seeds_in_store(move.side)
if move.index == 0: # pie move
self.our_side = Side.opposite(self.our_side)
self.side_to_move = MancalaEnv.make_move(self.board, move,
self.north_moved)
if move.side == Side.NORTH:
self.north_moved = True
seeds_in_store_after = self.board.get_seeds_in_store(move.side)
# Return a partial reward proportional to the number of captured seeds.
return (seeds_in_store_after - seeds_in_store_before) / 100.0
def __hash__(self) -> int:
primes = [
2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61,
67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137,
139, 149, 151, 157, 163
]
hashkey = 0
hashkey += primes[0] * Side.get_index(self.side_to_move)
hashkey += primes[1] * int(self.north_moved)
for hole in range(self.board.holes + 1):
hashkey += primes[2 + hole] * self.board.board[0][hole]
hashkey += primes[10 + hole] * self.board.board[1][hole]
return hashkey
def _run(self) -> Rollout:
# Choose randomly the side to play
self.trainer_side = Side.SOUTH if random.randint(
0, 1) == 0 else Side.NORTH
# Reset the environment so everything is in a clean state.
self.env.reset()
rollout = Rollout()
while not self.env.is_game_over():
# There is no choice if only one action is left. Taking that action automatically must be seen as
# a characteristic behaviour of the environment. This helped the learning of the agent
# to be more numerically stable (this is an empirical observation).
if len(self.env.get_legal_moves()) == 1:
action_left_to_perform = self.env.get_legal_moves()[0]
self.env.perform_move(action_left_to_perform)
continue
if self.env.side_to_move == self.trainer_side:
# If the agent is playing as NORTH, it's input would be a flipped board
flip_board = self.env.side_to_move == Side.NORTH
state = self.env.board.get_board_image(flipped=flip_board)
mask = self.env.get_action_mask_with_no_pie()
action, value = self.ac_net.sample(state, mask)
# Because the pie move with index 0 is ignored, the action indexes must be shifted by one
reward = self.env.perform_move(
Move(self.trainer_side, action + 1))
rollout.add(state, action, reward, value, mask)
else:
assert self.env.side_to_move == Side.opposite(
self.trainer_side)
action = self.opp_agent.produce_action(
self.env.board.get_board_image(),
self.env.get_action_mask_with_no_pie(),
self.env.side_to_move)
self.env.perform_move(Move(self.env.side_to_move, action + 1))
# We replace the partial reward of the last move with the final reward of the game
final_reward = self.env.compute_final_reward(self.trainer_side)
rollout.update_last_reward(final_reward)
if self.env.get_winner() == self.trainer_side:
rollout.add_win()
return rollout
def get_winner(self) -> Side or None:
"""
:return: The winning Side of the game or none if there is a tie.
"""
if not self.is_game_over():
raise ValueError(
'This method should be called only when the game is over')
finished_side = Side.NORTH if MancalaEnv.holes_empty(
self.board, Side.NORTH) else Side.SOUTH
not_finished_side = Side.opposite(finished_side)
not_finished_side_seeds = self.board.get_seeds_in_store(
not_finished_side)
for hole in range(1, self.board.holes + 1):
not_finished_side_seeds += self.board.get_seeds(
not_finished_side, hole)
finished_side_seeds = self.board.get_seeds_in_store(finished_side)
if finished_side_seeds > not_finished_side_seeds:
return finished_side
elif finished_side_seeds < not_finished_side_seeds:
return not_finished_side
return None
def __eq__(self, other):
return isinstance(other, self.__class__) \
and self.side == other.side \
and Side.get_index(self.side) == Side.get_index(other.side)
def get_seeds_in_store(self, side: Side):
return self.board[Side.get_index(side)][0]
def set_seeds_in_store(self, side: Side, seeds: int):
if seeds < 0:
raise ValueError('There has to be a non-negative number of seeds')
self.board[Side.get_index(side)][0] = seeds
def make_move(board: Board, move: Move, north_moved):
if not MancalaEnv.is_legal_action(board, move, north_moved):
raise ValueError(
'Move is illegal: Board: \n {} \n Move:\n {}/{} \n {}'.format(
board, move.index, move.side, north_moved))
# This is a pie move
if move.index == 0:
MancalaEnv.switch_sides(board)
return Side.opposite(move.side)
seeds_to_sow = board.get_seeds(move.side, move.index)
board.set_seeds(move.side, move.index, 0)
holes = board.holes
# Place seeds in all holes excepting the opponent's store
receiving_holes = 2 * holes + 1
# Rounds needed to sow all the seeds
rounds = seeds_to_sow // receiving_holes
# Seeds remaining after all the rounds
remaining_seeds = seeds_to_sow % receiving_holes
# Sow the seeds for the full rounds
if rounds != 0:
for hole in range(1, holes + 1):
board.add_seeds(Side.NORTH, hole, rounds)
board.add_seeds(Side.SOUTH, hole, rounds)
board.add_seeds_to_store(move.side, rounds)
# Sow the remaining seeds
sow_side = move.side
sow_hole = move.index
for _ in range(remaining_seeds):
sow_hole += 1
if sow_hole == 1:
sow_side = Side.opposite(sow_side)
if sow_hole > holes:
if sow_side == move.side:
sow_hole = 0
board.add_seeds_to_store(sow_side, 1)
continue
else:
sow_side = Side.opposite(sow_side)
sow_hole = 1
board.add_seeds(sow_side, sow_hole, 1)
# Capture the opponent's seeds from the opposite hole if the last seed
# is placed in an empty hole and there are seeds in the opposite hole
if sow_side == move.side and sow_hole > 0 \
and board.get_seeds(sow_side, sow_hole) == 1 \
and board.get_seeds_op(sow_side, sow_hole) > 0:
board.add_seeds_to_store(
move.side, 1 + board.get_seeds_op(sow_side, sow_hole))
board.set_seeds(move.side, sow_hole, 0)
board.set_seeds_op(move.side, sow_hole, 0)
# If the game is over, collect the seeds not in the store and put them there
game_over = MancalaEnv.game_over(board)
if game_over:
finished_side = Side.NORTH if MancalaEnv.holes_empty(
board, Side.NORTH) else Side.SOUTH
seeds = 0
collecting_side = Side.opposite(finished_side)
for hole in range(1, board.holes + 1):
seeds += board.get_seeds(collecting_side, hole)
board.set_seeds(collecting_side, hole, 0)
board.add_seeds_to_store(collecting_side, seeds)
# Return the side which is next to move
if sow_hole == 0 and (move.side == Side.NORTH or north_moved):
return move.side # Last seed was placed in the store, so side moves again
return Side.opposite(move.side)
def compute_final_reward(self, side: Side):
"""Returns a reward for the specified side for moving to the current state."""
reward = self.board.get_seeds_in_store(
side) - self.board.get_seeds_in_store(Side.opposite(side))
return reward
def get_seeds(self, side: Side, hole: int) -> int:
if hole < 1 or hole > self.holes:
raise ValueError(
'Hole number must be between 1 and number of holes')
return self.board[Side.get_index(side)][hole]
def test_side_opposite_is_correct(self):
self.assertEqual(Side.opposite(Side.NORTH), Side.SOUTH)
self.assertEqual(Side.opposite(Side.SOUTH), Side.NORTH)
def test_side_index_is_correct(self):
self.assertEqual(Side.get_index(Side.NORTH), 0)
self.assertEqual(Side.get_index(Side.SOUTH), 1)
def _run_game(player: Player, state: MancalaEnv):
our_agent_states = []
their_agent_states = []
both_agent_states = []
our_side = Side.SOUTH
while True:
msg = protocol.read_msg()
try:
msg_type = protocol.get_msg_type(msg)
if msg_type == MsgType.START:
first = protocol.interpret_start_msg(msg)
if first:
move = player.get_play(state)
protocol.send_msg(protocol.create_move_msg(move.index))
else:
our_side = Side.NORTH
elif msg_type == MsgType.STATE:
move_turn = protocol.interpret_state_msg(msg)
if move_turn.move == 0:
our_side = Side.opposite(our_side)
move_to_perform = Move(state.side_to_move, move_turn.move)
observed_state = ObservedState(state=state,
action_taken=move_to_perform)
both_agent_states.append(observed_state)
if state.side_to_move == our_side:
our_agent_states.append(observed_state)
else:
their_agent_states.append(observed_state)
state.perform_move(move_to_perform)
if not move_turn.end:
if move_turn.again:
move = player.get_play(state)
# pie rule; optimal move is to swap
if move.index == 0:
protocol.send_msg(protocol.create_swap_msg())
else:
protocol.send_msg(
protocol.create_move_msg(move.index))
elif msg_type == MsgType.END:
args = parser.parse_args()
run_id = '%06d' % args.run_number
run_category = args.category
_our_agent_file_path = _checkpoint_file_path + "/our-agent/" + run_category + run_id
_their_agent_file_path = _checkpoint_file_path + "/their-agent/" + run_category + run_id
_both_agent_file_path = _checkpoint_file_path + "/both-agent/" + run_category + run_id
np.save(file=_our_agent_file_path,
arr=np.array(our_agent_states))
np.save(file=_their_agent_file_path,
arr=np.array(their_agent_states))
np.save(file=_both_agent_file_path,
arr=np.array(both_agent_states))
break
else:
print("Not sure what I got " + str(msg_type))
except InvalidMessageException as _e:
print(str(_e))
def __hash__(self) -> int:
return self.index + (Side.get_index(self.side) * 10)
def __repr__(self) -> str:
return "Side: %s; Hole: %d" % (Side.side_to_str(self.side), self.index)
def get_seeds_op(self, side: Side, hole: int):
if hole < 1 or hole > self.holes:
raise ValueError(
'Hole number must be between 1 and number of holes')
return self.board[Side.get_index(Side.opposite(side))][self.holes + 1 -
hole]
