def policy_rollout(self, game_state):
for step in range(self.rollout_limit):
if game_state.is_over():
break
move_probabilities = self.rollout_policy.predict(game_state)
encoder = self.rollout_policy.encoder
legal_moves = game_state.legal_moves()
valid_moves = [
m for idx, m in enumerate(move_probabilities)
if Move(encoder.decode_point_index(idx)) in legal_moves
]
max_index, max_value = max(enumerate(valid_moves),
key=operator.itemgetter(1))
max_point = encoder.decode_point_index(max_index)
greedy_move = Move(max_point)
if greedy_move in legal_moves:
game_state = game_state.apply_move(greedy_move)
next_player = game_state.next_player
winner = game_state.winner()
if winner is not None:
return 1 if winner == next_player else -1
else:
return 0
def decode_move_index(self, index):
if index == self.board_size * self.board_size:
return Move.pass_turn()
row = index // self.board_size
col = index % self.board_size
return Move.play(Point(row=row + 1, col=col + 1))
def handle_play(self, color, move):
if move.lower() == 'pass':
self.game_state = self.game_state.apply_move(Move.pass_turn())
elif move.lower() == 'resign':
self.game_state = self.game_state.apply_move(Move.resign())
else:
self.game_state = self.game_state.apply_move(gtp_position_to_coords(move))
return response.success()
def process_zip(self, zip_file_name, data_file_name, game_list):
tar_file = self.unzip_data(zip_file_name)
zip_file = tarfile.open(self.data_dir + '/' + tar_file)
name_list = zip_file.getnames()
total_examples = self.num_total_examples(zip_file, game_list,
name_list)
shape = self.encoder.shape()
feature_shape = np.insert(shape, 0, np.asarray([total_examples]))
features = np.zeros(feature_shape)
labels = np.zeros((total_examples, ))
counter = 0
for index in game_list:
name = name_list[index + 1]
if not name.endswith('.sgf'):
raise ValueError(name + ' is not a valid sgf')
sgf_content = zip_file.extractfile(name).read()
sgf = Sgf_game.from_string(sgf_content)
game_state, first_move_done = self.get_handicap(sgf)
for item in sgf.main_sequence_iter():
color, move_tuple = item.get_move()
point = None
if color is not None:
if move_tuple is not None:
row, col = move_tuple
point = Point(row + 1, col + 1)
move = Move.play(point)
else:
move = Move.pass_turn()
if first_move_done and point is not None:
features[counter] = self.encoder.encode(game_state)
labels[counter] = self.encoder.encode_point(point)
counter += 1
game_state = game_state.apply_move(move)
first_move_done = True
feature_file_base = self.data_dir + '/' + data_file_name + '_features_%d'
label_file_base = self.data_dir + '/' + data_file_name + '_labels_%d'
chunk = 0 # Due to files with large content, split up after chunksize
chunksize = 1024
while features.shape[0] >= chunksize:
feature_file = feature_file_base % chunk
label_file = label_file_base % chunk
chunk += 1
current_features, features = features[:chunksize], features[
chunksize:]
current_labels, labels = labels[:chunksize], labels[chunksize:]
np.save(feature_file, current_features)
np.save(label_file, current_labels)
def test_new_game(self):
start = GameState.new_game(19)
next_state = start.apply_move(Move.play(Point(16, 16)))
self.assertEqual(start, next_state.previous_state)
self.assertEqual(Player.white, next_state.next_player)
self.assertEqual(Player.black, next_state.board.get(Point(16, 16)))
def encode(self, game_state, board_ext):
board_tensor = np.zeros(self.shape())
base_plane = {
game_state.next_player: 0,
game_state.next_player.other: 3
}
#Modify Board_ext for stones where value cost == 1 or cost == -1
try:
if len(board_ext._grid_ext) > 4:
max_value, point_black = board_ext.find_max_value()
go_string_black = game_state.board.get_go_string(point_black)
for point in go_string_black.stones:
board_ext._grid_ext[point][2] = 1
except:
pass
try:
if len(board_ext._grid_ext) > 4:
min_value, point_white = board_ext.find_min_value()
go_string_white = game_state.board.get_go_string(point_white)
for point in go_string_white.stones:
board_ext._grid_ext[point][2] = -1
except:
pass
# for row in range(self.board_height):
# for col in range(self.board_width):
# p = Point(row=row + 1, col=col + 1)
# go_string = game_state.board.get_go_string(p)
for row in range(self.board_height):
for col in range(self.board_width):
p = Point(row=row + 1, col=col + 1)
go_string = game_state.board.get_go_string(p)
if go_string is None:
if game_state.does_move_violate_ko(game_state.next_player,
Move.play(p)):
board_tensor[6][row][
col] = 1 # <1> For Ko situation both is equal
else:
liberty_plane = min(3, go_string.num_liberties) - 1
liberty_plane += base_plane[go_string.color]
if board_ext._grid_ext[p][2] < 0: # for white stones
if board_ext._grid_ext[p][2] > -0.5:
board_tensor[liberty_plane][row][col] = 0.5
else:
board_tensor[liberty_plane][row][
col] = -board_ext._grid_ext[p][2]
if board_ext._grid_ext[p][2] > 0: # for black stones
if board_ext._grid_ext[p][2] < 0.5:
board_tensor[liberty_plane][row][col] = 0.5
else:
board_tensor[liberty_plane][row][
col] = board_ext._grid_ext[p][2]
return board_tensor
def encode(self, game_state):
board_tensor = np.zeros(self.shape())
next_player = game_state.next_player
if game_state.next_player == Player.white:
board_tensor[8] = 1
else:
board_tensor[9] = 1
for r in range(self.board_size):
for c in range(self.board_size):
p = Point(row=r + 1, col=c + 1)
go_string = game_state.board.get_go_string(p)
if go_string is None:
if game_state.does_move_violate_ko(next_player,
Move.play(p)):
board_tensor[10][r][c] = 1
else:
liberty_plane = min(4, go_string.num_liberties) - 1
if go_string.color != next_player:
liberty_plane += 4
board_tensor[liberty_plane][r][c] = 1
return board_tensor
def play_their_move(self):
their_name = self.their_color.name
their_letter = their_name[0].upper()
pos = self.command_and_response("genmove {}\n".format(their_name))
if pos.lower() == 'resign':
self.game_state = self.game_state.apply_move(Move.resign())
self._stopped = True
elif pos.lower() == 'pass':
self.game_state = self.game_state.apply_move(Move.pass_turn())
self.sgf.append(";{}[]\n".format(their_letter))
if self.game_state.last_move.is_pass:
self._stopped = True
else:
move = gtp_position_to_coords(pos)
self.game_state = self.game_state.apply_move(move)
self.sgf.append(";{}[{}]\n".format(their_letter, self.sgf.coordinates(move)))
def legal_moves(state):
candidates = []
for r in range(1, state.board.num_rows + 1):
for c in range(1, state.board.num_cols + 1):
candidate = Move.play(Point(row=r, col=c))
if state.is_valid_move(candidate) \
and not is_point_an_eye(state.board, candidate.point, state.next_player):
candidates.append(candidate)
return candidates
def select_move(self, game_state):
"""Choose a random valid move that preserves our own eyes."""
candidates = []
for r in range(1, game_state.board.num_rows + 1):
for c in range(1, game_state.board.num_cols + 1):
candidate = Point(row=r, col=c)
print("kkk", candidate.row, candidate.col)
if game_state.is_valid_move(Move.play(candidate)) and \
not is_point_an_eye(game_state.board,
candidate,
game_state.next_player):
candidates.append(candidate)
if not candidates:
return Move.pass_turn()
return Move.play(random.choice(candidates))
# end::random_bot[]
def encode(self, game_state, board_ext=None):
board_tensor = np.zeros(self.shape())
#Modify Board_ext for stones where value cost == 1 or cost == -1
try:
if len(board_ext._grid_ext) > 4:
max_value, point_black = board_ext.find_max_value()
go_string_black = game_state.board.get_go_string(point_black)
for point in go_string_black.stones:
board_ext._grid_ext[point][2] = 1
except:
pass
try:
if len(board_ext._grid_ext) > 4:
min_value, point_white = board_ext.find_min_value()
go_string_white = game_state.board.get_go_string(point_white)
for point in go_string_white.stones:
board_ext._grid_ext[point][2] = -1
except:
pass
for row in range(self.board_height):
for col in range(self.board_width):
p = Point(row=row + 1, col=col + 1)
go_string = game_state.board.get_go_string(p)
if go_string is None:
if game_state.does_move_violate_ko(game_state.next_player,
Move.play(p)):
board_tensor[4][row][
col] = 1 # <1> For Ko situation both is equal
else:
if board_ext._grid_ext[p][2] < 0: # for white stones
if board_ext._grid_ext[p][2] > -0.5:
board_tensor[2][row][col] = 0.5
else:
board_tensor[2][row][
col] = -board_ext._grid_ext[p][2]
liberty_white = 1 - 1.0 / (go_string.num_liberties + 1)
board_tensor[3][row][col] = liberty_white
if board_ext._grid_ext[p][2] > 0: # for black stones
if board_ext._grid_ext[p][2] < 0.5:
board_tensor[0][row][col] = 0.5
else:
board_tensor[0][row][col] = board_ext._grid_ext[p][
2]
liberty_black = 1 - 1.0 / (go_string.num_liberties + 1)
board_tensor[1][row][col] = liberty_black
return board_tensor
def __init__(self, game_state, parent=None, move=None):
self.game_state = game_state
self.parent = parent
self.move = move
self.children = []
self.win_count = {Player.black: 0, Player.white: 0}
self.num_rollouts = 0
self.unvisited_moves = legal_moves(game_state) + [Move.pass_turn()]
def game(self) -> GameState:
if not self._game:
black = batch_translate_labels_to_coordinates(self.initial_black)
white = batch_translate_labels_to_coordinates(self.initial_white)
move_points = batch_translate_labels_to_coordinates(self.moves)
player = Player.black if self.initial_player == 'b' else Player.white
board = Board(19, 19)
for b in black:
board.place_stone(Player.black, b)
for w in white:
board.place_stone(Player.white, w)
self._game = GameState(board, player, None, None)
for move_point in move_points:
move = Move.pass_turn() if not move_point else Move.play(
move_point)
self._game = self._game.apply_move(move)
return self._game
def test_encoder(self):
alphago = AlphaGoEncoder()
start = GameState.new_game(19)
next_state = start.apply_move(Move.play(Point(16, 16)))
alphago.encode(next_state)
self.assertEquals(alphago.name(), 'alphago')
self.assertEquals(alphago.board_height, 19)
self.assertEquals(alphago.board_width, 19)
self.assertEquals(alphago.num_planes, 49)
self.assertEquals(alphago.shape(), (49, 19, 19))
def select_move(self, game_state):
best_moves = []
best_so_far = -MAX_SCORE
for move in legal_moves(game_state):
next_state = game_state.apply_move(move)
opponent_best = alpha_beta_best(next_state, best_so_far, self.max_depth, self.eval_fn)
# opponent_best = best_result(next_state, self.max_depth, self.eval_fn)
our_result = -opponent_best
if our_result > best_so_far:
best_moves = [move]
best_so_far = our_result
elif our_result == best_so_far:
best_moves.append(move)
if not best_moves:
return Move.pass_turn()
return random.choice(best_moves)
def encode(self, game_state):
board_tensor = np.zeros(self.shape())
base_plane = {
game_state.next_player: 0,
game_state.next_player.other: 3
}
for row in range(self.board_height):
for col in range(self.board_width):
p = Point(row=row + 1, col=col + 1)
go_string = game_state.board.get_go_string(p)
if go_string is None:
if game_state.does_move_violate_ko(game_state.next_player,
Move.play(p)):
board_tensor[6][row][col] = 1 # <1>
else:
liberty_plane = min(3, go_string.num_liberties) - 1
liberty_plane += base_plane[go_string.color]
board_tensor[liberty_plane][row][col] = 1 # <2>
return board_tensor
def main():
board_size = 4
game = GameState.new_game(board_size)
bot = MinimaxBot(5, capture_diff)
while not game.is_over():
print(chr(27) + "[2J")
print_board(game.board)
if game.next_player == Player.black:
valid = False
while not valid:
human_move = input('-- ')
human_move = human_move.upper()
point = point_from_coords(human_move.strip())
move = Move.play(point)
valid = game.is_valid_move(move)
else:
move = bot.select_move(game)
print_move(game.next_player, move)
game = game.apply_move(move)
def policy_rollout(self, game_state):
for step in range(self.rollout_limit):
if game_state.is_over():
break
move_probabilities = self.rollout_policy.predict(game_state)
encoder = self.rollout_policy.encoder
for idx in np.argsort(move_probabilities)[::-1]:
max_point = encoder.decode_point_index(idx)
greedy_move = Move(max_point)
if greedy_move in game_state.legal_moves():
game_state = game_state.apply_move(greedy_move)
break
next_player = game_state.next_player
winner = game_state.winner()
if winner is not None:
return 1 if winner == next_player else -1
else:
return 0
def process_zip(self, zip_file_name, data_file_name, game_list):
# total number of moves in all games in zip file
tar_file = self.unzip_data(zip_file_name)
zip_file = tarfile.open(self.data_dir + '/' + tar_file)
name_list = zip_file.getnames()
total_examples = self.num_total_examples(zip_file, game_list,
name_list)
# infers shape of features and labels from the encoder
shape = self.encoder.shape() # <2>
feature_shape = np.insert(shape, 0, np.asarray([total_examples]))
features = np.zeros(feature_shape)
labels = np.zeros((total_examples, ))
counter = 0
for index in game_list:
# reads the SGF contents as a string
name = name_list[index + 1]
if not name.endswith('.sgf'):
raise ValueError(name + ' is not a valid sgf')
sgf_content = zip_file.extractfile(name).read()
sgf = SgfGame.from_string(sgf_content)
# apply handicap stones
game_state, first_move_done = self.get_handicap(sgf)
# iterates through all moves
for item in sgf.main_sequence_iter():
color, move_tuple = item.get_move()
point = None
if color is not None:
# read the coordinate of the stone to be played
if move_tuple is not None:
row, col = move_tuple
point = Point(row + 1, col + 1)
move = Move.play(point)
else:
# or do nothing
move = Move.pass_turn()
if first_move_done and point is not None:
# Encode the current games state as features
features[counter] = self.encoder.encode(game_state)
# Encode the next move as label
labels[counter] = self.encoder.encode_point(point)
counter += 1
# Apply the move and move on to the next one.
game_state = game_state.apply_move(move)
first_move_done = True
# Finish implementation of process_zip
feature_file_base = self.data_dir + '/' + data_file_name + '_features_%d'
label_file_base = self.data_dir + '/' + data_file_name + '_labels_%d'
chunk = 0 # Due to files with large content, split up after chunksize
chunksize = 1024
# author's code which doesn't do what he thinks it does
# # Process features and labels in chunks of 1024
# while features.shape[0] >= chunksize:
# feature_file = feature_file_base % chunk
# label_file = label_file_base % chunk
# chunk += 1
# # break up chunk into another piece
# current_features, features = features[:chunksize], features[chunksize:]
# current_labels, labels = labels[:chunksize], labels[chunksize:]
# # store into a separate file
# np.save(feature_file, current_features)
# np.save(label_file, current_labels)
# fixed code:
while features.shape[0] > 0:
feature_file = feature_file_base % chunk
label_file = label_file_base % chunk
chunk += 1
current_features, features = features[:chunksize], features[
chunksize:]
current_labels, labels = labels[:chunksize], labels[chunksize:]
np.save(feature_file, current_features)
np.save(label_file, current_labels)
def gtp_position_to_coords(gtp_position):
col_str, row_str = gtp_position[0], gtp_position[1:]
point = Point(int(row_str), COLS.find(col_str.upper()) + 1)
return Move(point)
def encode(self, game_state):
board_tensor = np.zeros(
(self.num_planes, self.board_height, self.board_width))
for r in range(self.board_height):
for c in range(self.board_width):
point = Point(row=r + 1, col=c + 1)
go_string = game_state.board.get_go_string(point)
if go_string and go_string.color == game_state.next_player:
board_tensor[offset("stone_color")][r][c] = 1
elif go_string and go_string.color == game_state.next_player.other:
board_tensor[offset("stone_color") + 1][r][c] = 1
else:
board_tensor[offset("stone_color") + 2][r][c] = 1
board_tensor[offset("ones")] = self.ones()
board_tensor[offset("zeros")] = self.zeros()
if not is_point_an_eye(game_state.board, point,
game_state.next_player):
board_tensor[offset("sensibleness")][r][c] = 1
##ages = min(game_state.board.move_ages.get(r, c, step?), 8)
ages = int(min(game_state.board.move_ages.get(r, c),
8)) # Nail insert int(...)
if ages > 0:
#print(ages)
board_tensor[offset("turns_since") + ages][r][c] = 1
if game_state.board.get_go_string(point):
liberties = min(
game_state.board.get_go_string(point).num_liberties, 8)
board_tensor[offset("liberties") + liberties][r][c] = 1
move = Move(point)
if game_state.is_valid_move(move):
new_state = game_state.apply_move(move)
liberties = min(
new_state.board.get_go_string(point).num_liberties, 8)
board_tensor[offset("liberties_after") +
liberties][r][c] = 1
adjacent_strings = [
game_state.board.get_go_string(nb)
for nb in point.neighbors()
]
capture_count = 0
for go_string in adjacent_strings:
other_player = game_state.next_player.other
if go_string and go_string.num_liberties == 1 and go_string.color == other_player:
capture_count += len(go_string.stones)
capture_count = min(capture_count, 8)
board_tensor[offset("capture_size") +
capture_count][r][c] = 1
if go_string and go_string.num_liberties == 1:
go_string = game_state.board.get_go_string(point)
if go_string:
num_atari_stones = min(len(go_string.stones), 8)
board_tensor[offset("self_atari_size") +
num_atari_stones][r][c] = 1
if is_ladder_capture(game_state, point):
board_tensor[offset("ladder_capture")][r][c] = 1
if is_ladder_escape(game_state, point):
board_tensor[offset("ladder_escape")][r][c] = 1
if self.use_player_plane:
if game_state.next_player == Player.black:
board_tensor[offset("ones")] = self.ones()
else:
board_tensor[offset("zeros")] = self.zeros()
return board_tensor
def command(self, move: Move = None) -> Tuple[Command, int]:
# Get the game that reflects the passed move having been played (if supplied).
game = self.game if not move else self.game.apply_move(move)
# Process the game board to get the necessary information to generate canonical encodings.
point_plus_code: List[Tuple[Point, int]] = []
for i in intersections:
color = game.board.get(i)
if not color:
code = 0 if game.is_valid_move(Move.play(i)) else 3
else:
code = 1 if color == Player.black else 2
if code:
point_plus_code.append((i, code))
# Select the transformation that leads to the lowest canonical position representation.
selected_form = float('inf')
selected_ordinal = -1
selected_transformation = None
for ordinal, transformation in enumerate(transformations):
encoding = self._encode_point_colorings(point_plus_code,
transformation)
if encoding < selected_form:
selected_form = encoding
selected_ordinal = ordinal
selected_transformation = transformation
# Encode the resulting board position as a string.
position_representation = self._convert_code_to_dense_string(
selected_form)
# Transform the starting stone points using the selected transformation.
initial_positions_plus_colors: List[Tuple[Point, int]] = []
initial_stones: List[Move] = []
if self.initial_black:
transformed_black_points = [
selected_transformation(translate_label_to_point(x))
for x in self.initial_black
]
initial_positions_plus_colors += [
(x, 1) for x in transformed_black_points
]
initial_stones += [
MoveInfo(KataGoPlayer.b, coords_from_point(x))
for x in transformed_black_points
]
if self.initial_white:
transformed_white_points = [
selected_transformation(translate_label_to_point(x))
for x in self.initial_white
]
initial_positions_plus_colors += [
(x, 2) for x in transformed_white_points
]
initial_stones += [
MoveInfo(KataGoPlayer.w, coords_from_point(x))
for x in transformed_white_points
]
initial_form = self._encode_point_colorings(
initial_positions_plus_colors)
initial_representation = self._convert_code_to_dense_string(
initial_form)
# Compose an ID to use when communicating with KataGo. Because it is possible to arrive at the same position
# in multiple paths and/or transformations, this ID does NOT contain the information necessary to return to the
# original representation. That exists for communicating between the UI and the server ONLY.
next_player = "b" if game.next_player == Player.black else "w"
id = f'{self.ruleset}_{self.komi}_{next_player}_{initial_representation}_{position_representation}'
# Build the command!
command = Command()
command.id = id
command.komi = self.komi
command.initialPlayer = KataGoPlayer.b if self.initial_player == 'b' else KataGoPlayer.w
command.rules = self.ruleset
command.initialStones = initial_stones
command.moves = []
player = command.initialPlayer
for move in game.history:
move_info = MoveInfo(
player,
'pass' if not move or move.is_pass else coords_from_point(
selected_transformation(move.point)))
command.moves.append(move_info)
player = player.opposite
command.analyzeTurns = [len(command.moves)]
return command, selected_ordinal
from dlgo.gosgf import Sgf_game
from dlgo.goboard_fast import GameState, Move
from dlgo.gotypes import Point
from dlgo.utils import print_board
sgf_content = "(;GM[1]FF[4]SZ[9];B[ee];W[ef];B[ff]" + \
";W[df];B[fe];W[fc];B[ec];W[gd];B[fb])"
sgf_game = Sgf_game.from_string(sgf_content)
game_state = GameState.new_game(19)
for item in sgf_game.main_sequence_iter():
color, move_tuple = item.get_move()
if color is not None and move_tuple is not None:
row, col = move_tuple
point = Point(row + 1, col + 1)
move = Move.play(point)
game_state = game_state.apply_move(move)
print_board(game_state.board)
def select_move(self, game_state):
candidates = legal_moves(game_state)
if not candidates:
return Move.pass_turn()
return random.choice(candidates)
def process_zip(self, zip_file_name, data_file_name, game_list):
# total number of moves in all games in zip file
tar_file = self.unzip_data(zip_file_name)
zip_file = tarfile.open(self.data_dir + '/' + tar_file)
name_list = zip_file.getnames()
total_examples = self.num_total_examples(zip_file, game_list, name_list)
new_game_list = game_list.copy()
shape = self.encoder.shape()
# Changed code to prevent too big features in memory
# Finish implementation of process_zip
feature_file_base = self.data_dir + '/' + data_file_name + '_features_%d'
label_file_base = self.data_dir + '/' + data_file_name + '_labels_%d'
chunk = 0
examples_used = 1024
iters = math.ceil(total_examples / examples_used)
for z in range(iters):
# <2>
#feature_shape = np.insert(shape, 0, np.asarray([examples_used]))
# features = np.zeros(feature_shape)
# labels = np.zeros((examples_used,))
# counter = 0
for index in new_game_list[:examples_used]:
features = []
labels = []
# reads the SGF contents as a string
name = name_list[index + 1]
if not name.endswith('.sgf'):
raise ValueError(name + ' is not a valid sgf')
sgf_content = zip_file.extractfile(name).read()
sgf = SgfGame.from_string(sgf_content)
# apply handicap stones
game_state, first_move_done = self.get_handicap(sgf)
# iterates through all moves
for item in sgf.main_sequence_iter():
z = np.zeros(shape)
color, move_tuple = item.get_move()
point = None
if color is not None:
# read the coordinate of the stone to be played
if move_tuple is not None:
row, col = move_tuple
point = Point(row + 1, col + 1)
move = Move.play(point)
else:
# or do nothing
move = Move.pass_turn()
if first_move_done and point is not None:
# Encode the current games state as features
# features[counter] = self.encoder.encode(game_state)
z = self.encoder.encode(game_state)
features.append(z)
# Encode the next move as label
# labels[counter] = self.encoder.encode_point(point)
labels.append(self.encoder.encode_point(point))
# counter += 1
# Apply the move and move on to the next one.
game_state = game_state.apply_move(move)
first_move_done = True
new_game_list = new_game_list[examples_used:]
feature_file = feature_file_base % chunk
label_file = label_file_base % chunk
chunk += 1
# current_features, features = features[:examples_used], features[examples_used:]
# current_labels, labels = labels[:examples_used], labels[examples_used:]
# np.save(feature_file, current_features)
# np.save(label_file, current_labels)
features = np.array(features)
labels = np.array(labels)
np.save(feature_file, features)
np.save(label_file, labels)
#
"""
def process_zip(self, zip_file_name, data_file_name, game_list):
tar_file = self.unzip_data(zip_file_name)
zip_file = tarfile.open(self.data_dir + '/' + tar_file)
name_list = zip_file.getnames()
total_examples = self.num_total_examples(zip_file, game_list,
name_list)
shape = self.encoder.shape()
feature_shape = np.insert(shape, 0, np.asarray([total_examples]))
features = np.zeros(feature_shape)
labels = np.zeros((total_examples, ))
counter = 0
board = Board(19, 19) #Nail
board_ext = Board_Ext(board) # Nail
for index in game_list:
name = name_list[index + 1]
if not name.endswith('.sgf'):
raise ValueError(name + ' is not a valid sgf')
sgf_content = zip_file.extractfile(name).read()
sgf = Sgf_game.from_string(sgf_content)
game_state, first_move_done, board_ext = self.get_handicap(sgf)
# if first_move_done : # Nail ignore handicap
# continue # Ignore games with handicap
if self.encoder.name(
)[:2] == 'my' and first_move_done == False: # Not handicap
board_ext = Board_Ext(game_state.board) #inserted Nail
for item in sgf.main_sequence_iter():
color, move_tuple = item.get_move()
point = None
if color is not None:
if move_tuple is not None:
row, col = move_tuple
point = Point(row + 1, col + 1)
move = Move.play(point)
else:
move = Move.pass_turn()
if first_move_done and point is not None:
encode = True
# # Data only for debute Nail
# if self.count_stones_debute is not None and \
# self.count_stones_middle is None and \
# self.count_stones_end is None and \
# board_ext.count_stones() > self.count_stones_debute:
# encode = False
# # Data for middle game Nail
# if self.count_stones_debute is not None and \
# self.count_stones_middle is not None and \
# self.count_stones_end is None and \
# self.board_ext.count_stones() <= self.count_stones_debute and board_ext.count_stones() > self.count_stones_middle:
# encode = False
#
# # Data for end
# if self.count_stones_middle is not None and \
# self.count_stones_end is not None and \
# self.board_ext.count_stones() <= self.count_stones_middle and board_ext.count_stones() > self.count_stones_end:
# encode = False
if encode == True: #Nail
if self.encoder.name()[:2] == 'my':
features[counter] = self.encoder.encode(
game_state, board_ext) #Nail
else:
features[counter] = self.encoder.encode(
game_state)
labels[counter] = self.encoder.encode_point(point)
counter += 1
game_state = game_state.apply_move(move)
if self.encoder.name()[:2] == 'my':
board_ext.place_stone_ext(game_state.board, color,
point) # Inserted Nail
# Nail
first_move_done = True
feature_file_base = self.data_dir + '/' + data_file_name + '_features_%d'
label_file_base = self.data_dir + '/' + data_file_name + '_labels_%d'
chunk = 0 # Due to files with large content, split up after chunksize
chunksize = 1024
start_time_all = time.time()
while features.shape[0] >= chunksize:
start_time = time.time()
feature_file = feature_file_base % chunk
label_file = label_file_base % chunk
chunk += 1
current_features, features = features[:chunksize], features[
chunksize:]
current_labels, labels = labels[:chunksize], labels[chunksize:]
np.save(feature_file, current_features)
np.save(label_file, current_labels)
# Inserted Nail
print("Chunk = ", chunk, " File for training Current_features: ",
feature_file)
print("Time per one file = ",
(time.time() - start_time_all) / 1000, ' seconds')
print('Files preparation with proccess_zip is over\n')
print('Full Time = ', (time.time() - start_time_all) / 1000,
' seconds')
print("End chunk = ", chunk)
def process_zip(self, zip_file_name, data_file_name, game_list):
feature_file_base = self.data_dir + '/' + data_file_name + '_features_%d'
label_file_base = self.data_dir + '/' + data_file_name + '_labels_%d'
print("processing {}...".format(data_file_name))
tar_file = self.unzip_data(zip_file_name)
zip_file = tarfile.open(self.data_dir + '/' + tar_file)
name_list = zip_file.getnames()
total_examples = self.num_total_examples(zip_file, game_list,
name_list)
shape = self.encoder.shape()
feature_shape = np.insert(shape, 0, np.asarray([total_examples]))
features = np.zeros(feature_shape)
labels = np.zeros((total_examples, ))
counter = 0
for index in game_list:
name = name_list[index + 1]
if not name.endswith('.sgf'):
raise ValueError(name + ' is not a valid sgf')
sgf_content = zip_file.extractfile(name).read()
sgf = SgfGame.from_string(sgf_content)
game_state, first_move_done = self.get_handicap(sgf)
for item in sgf.main_sequence_iter():
color, move_tuple = item.get_move()
point = None
if color is not None:
if move_tuple is not None:
row, col = move_tuple
point = Point(row + 1, col + 1)
move = Move.play(point)
else:
move = Move.pass_turn()
if first_move_done and point is not None:
features[counter] = self.encoder.encode(game_state)
labels[counter] = self.encoder.encode_point(point)
counter += 1
game_state = game_state.apply_move(move)
first_move_done = True
chunk = 0
chunksize = 1024
# Does not do what author thinks it does -- truncates data
# while features.shape[0] >= chunksize:
# feature_file = feature_file_base % chunk
# label_file = label_file_base % chunk
# chunk += 1
# current_features, features = features[:chunksize], features[chunksize:]
# current_labels, labels = labels[:chunksize], labels[chunksize:]
# np.save(feature_file, current_features)
# np.save(label_file, current_labels)
# fixed code:
while features.shape[0] > 0:
feature_file = feature_file_base % chunk
label_file = label_file_base % chunk
chunk += 1
current_features, features = features[:chunksize], features[
chunksize:]
current_labels, labels = labels[:chunksize], labels[chunksize:]
np.save(feature_file, current_features)
print('wrote {}'.format(feature_file))
np.save(label_file, current_labels)
print('wrote {}'.format(label_file))
# allow garbage collection
features = None
labels = None
current_features = None
current_labels = None
def process_zip(self, zip_file_name, data_file_name, game_list):
tar_file = self.unzip_data(zip_file_name)
zip_file = tarfile.open(self.data_dir + '/' + tar_file)
name_list = zip_file.getnames()
# このzipファイル内の全てのゲームの合計着手回数を決定する
total_examples = self.num_total_examples(zip_file, game_list,
name_list) # <1>
# 使用するエンコーダからのフィーちゃとラベルの形状を推測する
shape = self.encoder.shape() # <2>
feature_shape = np.insert(shape, 0, np.asarray([total_examples]))
features = np.zeros(feature_shape)
labels = np.zeros((total_examples, ))
counter = 0
for index in game_list:
name = name_list[index + 1]
if not name.endswith('.sgf'):
raise ValueError(name + ' is not a valid sgf')
sgf_content = zip_file.extractfile(name).read()
# zipファイルを解凍した後、SGFの内容を文字列として読み込む
sgf = Sgf_game.from_string(sgf_content) # <3>
# すべての置石を適用して、初期のゲーム状態を推測する
game_state, first_move_done = self.get_handicap(sgf) # <4>
# SGFファイル内のすべての着手を繰り返す
for item in sgf.main_sequence_iter(): # <5>
color, move_tuple = item.get_move()
point = None
if color is not None:
# 着手する石の座標を読み込み
if move_tuple is not None: # <6>
row, col = move_tuple
point = Point(row + 1, col + 1)
move = Move.play(point)
else:
# ない場合はパス
move = Move.pass_turn() # <7>
if first_move_done and point is not None:
# 現在のゲームの状態を特徴量としてエンコード
features[counter] = self.encoder.encode(
game_state) # <8>
# 次の着手を特徴量に対するラベルとしてエンコードする
labels[counter] = self.encoder.encode_point(
point) # <9>
counter += 1
# その後、着手を盤に適用し、次に進む
game_state = game_state.apply_move(move) # <10>
first_move_done = True
# <1> Determine the total number of moves in all games in this zip file.
# <2> Infer the shape of features and labels from the encoder we use.
# <3> Read the SGF content as string, after extracting the zip file.
# <4> Infer the initial game state by applying all handicap stones.
# <5> Iterate over all moves in the SGF file.
# <6> Read the coordinates of the stone to be played...
# <7> ... or pass, if there is none.
# <8> We encode the current game state as features...
# <9> ... and the next move as label for the features.
# <10> Afterwards the move is applied to the board and we proceed with the next one.
# end::read_sgf_files[]
# tag::store_features_and_labels[]
# 特徴量とラベルを小さなチャンクとしてローカルに保持する
# 小さなチャンクを格納する理由は、データの配列が非常に高速になり、後でより柔軟な小さなファイルにデータを格納できるため。
feature_file_base = self.data_dir + '/' + data_file_name + '_features_%d'
label_file_base = self.data_dir + '/' + data_file_name + '_labels_%d'
chunk = 0 # Due to files with large content, split up after chunksize
chunksize = 1024
# 特徴量とラベルを1024のサイズのチャンクで処理する
while features.shape[0] >= chunksize: # <1>
feature_file = feature_file_base % chunk
label_file = label_file_base % chunk
chunk += 1
current_features, features = features[:chunksize], features[
chunksize:]
# 現在のチャンクは特徴量とラベルから切り離されている
current_labels, labels = labels[:chunksize], labels[
chunksize:] # <2>
np.save(feature_file, current_features)
# 別々のファイルに保存される
np.save(label_file, current_labels) # <3>
def process_zip(self, zip_file_name, data_file_name, game_list):
"""
.tar.gzファイルを解凍し，
必要なゲームだけ特徴量とラベルに変換して任意の名前で保存
1024の着手データごとに一つのファイルに保存する
これにより，動的ロードによるメモリの節約ができる
Parameters
----------
zip_file_name : str
解凍対象となるファイルパス
data_file_name : str
特徴量とラベルの保存先パス
game_list : list
解凍対象から選ばれるゲームのインデックスのリスト
"""
# ファイルを解凍し，必要な棋譜データの数を取得
tar_file = self.unzip_data(zip_file_name)
zip_file = tarfile.open(self.data_dir + '/' + tar_file)
name_list = zip_file.getnames()
total_examples = self.num_total_examples(zip_file, game_list,
name_list)
# 空の特徴量とラベルを用意
shape = self.encoder.shape()
feature_shape = np.insert(shape, 0, total_examples)
features = np.zeros(feature_shape)
labels = np.zeros((total_examples, ))
# 必要な全てのゲームインデックスのゲームを再生しながら記録していく
counter = 0 # 着手数
for index in game_list:
# 対象となるsgfファイルの棋譜データを読み込み
name = name_list[index + 1]
if not name.endswith('.sgf'):
raise ValueError(name + ' is not a valid sgf')
# メンバーをファイルオブジェクトとして抽出
sgf_content = zip_file.extractfile(name).read()
sgf = Sgf_game.from_string(sgf_content)
# ハンディキャップの適用
game_state, first_move_done = self.get_handicap(sgf)
# 対局再生
for item in sgf.main_sequence_iter():
color, move_tuple = item.get_move()
# 着手
if color is not None:
# 打石
if move_tuple is not None:
row, col = move_tuple
point = Point(row + 1, col + 1)
move = Move.play(point)
# パス
else:
move = Move.pass_turn()
# 初手は盤面が空である．空の盤面はデータに加えない．
if first_move_done:
# 現在の盤面を特徴量として，
features[counter] = self.encoder.encode(game_state)
# その盤面に対するこのターンの着手をラベルとして記録
labels[counter] = self.encoder.encode_point(point)
# 着手数をカウント
counter += 1
# 着手を適用
game_state = game_state.apply_move(move)
first_move_done = True
# 保存するファイル名のプレースホルダ
feature_file_base = self.data_dir + '/' + data_file_name + '_features_%d'
label_file_base = self.data_dir + '/' + data_file_name + '_labels_%d'
# 全てのデータを一つに保存するのではなく，chunk_sizeで区切って保存
chunk = 0
chunk_size = 1024
while features.shape[0] >= chunk_size:
feature_file = feature_file_base % chunk
label_file = label_file_base % chunk
chunk += 1
# chunk_sizeでfeaturesとlabelsを区切っていく
current_features, features = features[:chunk_size], features[
chunk_size:]
current_labels, labels = labels[:chunk_size], labels[chunk_size:]
# 区切ったfeaturesとlabelsを保存
np.save(feature_file, current_features)
np.save(label_file, current_labels)
def process_zip(self, zip_file_name, data_file_name, game_list):
tar_file = self.unzip_data(zip_file_name)
zip_file = tarfile.open(self.data_dir + '/' + tar_file)
name_list = zip_file.getnames()
total_examples = self.num_total_examples(zip_file, game_list,
name_list) # <1>
shape = self.encoder.shape() # <2>
feature_shape = np.insert(shape, 0, np.asarray([total_examples]))
features = np.zeros(feature_shape)
labels = np.zeros((total_examples, ))
counter = 0
for index in game_list:
name = name_list[index + 1]
if not name.endswith('.sgf'):
raise ValueError(name + ' is not a valid sgf')
sgf_content = zip_file.extractfile(name).read()
sgf = Sgf_game.from_string(sgf_content) # <3>
game_state, first_move_done = self.get_handicap(sgf) # <4>
for item in sgf.main_sequence_iter(): # <5>
color, move_tuple = item.get_move()
point = None
if color is not None:
if move_tuple is not None: # <6>
row, col = move_tuple
point = Point(row + 1, col + 1)
move = Move.play(point)
else:
move = Move.pass_turn() # <7>
if first_move_done and point is not None:
features[counter] = self.encoder.encode(
game_state) # <8>
labels[counter] = self.encoder.encode_point(
point) # <9>
counter += 1
game_state = game_state.apply_move(move) # <10>
first_move_done = True
# <1> Determine the total number of moves in all games in this zip file.
# <2> Infer the shape of features and labels from the encoder we use.
# <3> Read the SGF content as string, after extracting the zip file.
# <4> Infer the initial game state by applying all handicap stones.
# <5> Iterate over all moves in the SGF file.
# <6> Read the coordinates of the stone to be played...
# <7> ... or pass, if there is none.
# <8> We encode the current game state as features...
# <9> ... and the next move as label for the features.
# <10> Afterwards the move is applied to the board and we proceed with the next one.
# end::read_sgf_files[]
# tag::store_features_and_labels[]
feature_file_base = self.data_dir + '/' + data_file_name + '_features_%d'
label_file_base = self.data_dir + '/' + data_file_name + '_labels_%d'
chunk = 0 # Due to files with large content, split up after chunksize
chunksize = 1024
while features.shape[0] >= chunksize: # <1>
feature_file = feature_file_base % chunk
label_file = label_file_base % chunk
chunk += 1
current_features, features = features[:chunksize], features[
chunksize:]
current_labels, labels = labels[:chunksize], labels[
chunksize:] # <2>
np.save(feature_file, current_features)
np.save(label_file, current_labels) # <3>