def decode_point_index(self, index):
row = index // self.board_width
col = index % self.board_width
return Point(row=row + 1, col=col + 1)
def process_zip(self, zip_file_name, data_file_name, game_list):
tar_file = self.unzip_data(zip_file_name)
zip_file = tarfile.open(self.data_dir + '/' + tar_file)
name_list = zip_file.getnames()
# このzipファイル内の全てのゲームの合計着手回数を決定する
total_examples = self.num_total_examples(zip_file, game_list,
name_list) # <1>
# 使用するエンコーダからのフィーちゃとラベルの形状を推測する
shape = self.encoder.shape() # <2>
feature_shape = np.insert(shape, 0, np.asarray([total_examples]))
features = np.zeros(feature_shape)
labels = np.zeros((total_examples, ))
counter = 0
for index in game_list:
name = name_list[index + 1]
if not name.endswith('.sgf'):
raise ValueError(name + ' is not a valid sgf')
sgf_content = zip_file.extractfile(name).read()
# zipファイルを解凍した後、SGFの内容を文字列として読み込む
sgf = Sgf_game.from_string(sgf_content) # <3>
# すべての置石を適用して、初期のゲーム状態を推測する
game_state, first_move_done = self.get_handicap(sgf) # <4>
# SGFファイル内のすべての着手を繰り返す
for item in sgf.main_sequence_iter(): # <5>
color, move_tuple = item.get_move()
point = None
if color is not None:
# 着手する石の座標を読み込み
if move_tuple is not None: # <6>
row, col = move_tuple
point = Point(row + 1, col + 1)
move = Move.play(point)
else:
# ない場合はパス
move = Move.pass_turn() # <7>
if first_move_done and point is not None:
# 現在のゲームの状態を特徴量としてエンコード
features[counter] = self.encoder.encode(
game_state) # <8>
# 次の着手を特徴量に対するラベルとしてエンコードする
labels[counter] = self.encoder.encode_point(
point) # <9>
counter += 1
# その後、着手を盤に適用し、次に進む
game_state = game_state.apply_move(move) # <10>
first_move_done = True
# <1> Determine the total number of moves in all games in this zip file.
# <2> Infer the shape of features and labels from the encoder we use.
# <3> Read the SGF content as string, after extracting the zip file.
# <4> Infer the initial game state by applying all handicap stones.
# <5> Iterate over all moves in the SGF file.
# <6> Read the coordinates of the stone to be played...
# <7> ... or pass, if there is none.
# <8> We encode the current game state as features...
# <9> ... and the next move as label for the features.
# <10> Afterwards the move is applied to the board and we proceed with the next one.
# end::read_sgf_files[]
# tag::store_features_and_labels[]
# 特徴量とラベルを小さなチャンクとしてローカルに保持する
# 小さなチャンクを格納する理由は、データの配列が非常に高速になり、後でより柔軟な小さなファイルにデータを格納できるため。
feature_file_base = self.data_dir + '/' + data_file_name + '_features_%d'
label_file_base = self.data_dir + '/' + data_file_name + '_labels_%d'
chunk = 0 # Due to files with large content, split up after chunksize
chunksize = 1024
# 特徴量とラベルを1024のサイズのチャンクで処理する
while features.shape[0] >= chunksize: # <1>
feature_file = feature_file_base % chunk
label_file = label_file_base % chunk
chunk += 1
current_features, features = features[:chunksize], features[
chunksize:]
# 現在のチャンクは特徴量とラベルから切り離されている
current_labels, labels = labels[:chunksize], labels[
chunksize:] # <2>
np.save(feature_file, current_features)
# 別々のファイルに保存される
np.save(label_file, current_labels) # <3>
def test_scoring(self):
# .w.ww
# wwww.
# bbbww
# .bbbb
# .b.b.
board = Board(5, 5)
board.place_stone(Player.black, Point(1, 2))
board.place_stone(Player.black, Point(1, 4))
board.place_stone(Player.black, Point(2, 2))
board.place_stone(Player.black, Point(2, 3))
board.place_stone(Player.black, Point(2, 4))
board.place_stone(Player.black, Point(2, 5))
board.place_stone(Player.black, Point(3, 1))
board.place_stone(Player.black, Point(3, 2))
board.place_stone(Player.black, Point(3, 3))
board.place_stone(Player.white, Point(3, 4))
board.place_stone(Player.white, Point(3, 5))
board.place_stone(Player.white, Point(4, 1))
board.place_stone(Player.white, Point(4, 2))
board.place_stone(Player.white, Point(4, 3))
board.place_stone(Player.white, Point(4, 4))
board.place_stone(Player.white, Point(5, 2))
board.place_stone(Player.white, Point(5, 4))
board.place_stone(Player.white, Point(5, 5))
territory = scoring.evaluate_territory(board)
self.assertEqual(9, territory.num_black_stones)
self.assertEqual(4, territory.num_black_territory)
self.assertEqual(9, territory.num_white_stones)
self.assertEqual(3, territory.num_white_territory)
self.assertEqual(0, territory.num_dame)
def decode_point_index(self, index):
"""Turn an integer index into a board point."""
row = index // self.board_width
col = index % self.board_width
return Point(row=row + 1, col=col + 1)
def process_zip(self, zip_file_name, data_file_name, game_list):
tar_file = self.unzip_data(zip_file_name)
zip_file = tarfile.open(self.data_dir + '/' + tar_file)
name_list = zip_file.getnames()
total_examples = self.num_total_examples(zip_file, game_list,
name_list)
shape = self.encoder.shape()
feature_shape = np.insert(shape, 0, np.asarray([total_examples]))
features = np.zeros(feature_shape)
labels = np.zeros((total_examples, ))
counter = 0
board = Board(19, 19) #Nail
board_ext = Board_Ext(board) # Nail
for index in game_list:
name = name_list[index + 1]
if not name.endswith('.sgf'):
raise ValueError(name + ' is not a valid sgf')
sgf_content = zip_file.extractfile(name).read()
sgf = Sgf_game.from_string(sgf_content)
game_state, first_move_done, board_ext = self.get_handicap(sgf)
# if first_move_done : # Nail ignore handicap
# continue # Ignore games with handicap
if self.encoder.name(
)[:2] == 'my' and first_move_done == False: # Not handicap
board_ext = Board_Ext(game_state.board) #inserted Nail
for item in sgf.main_sequence_iter():
color, move_tuple = item.get_move()
point = None
if color is not None:
if move_tuple is not None:
row, col = move_tuple
point = Point(row + 1, col + 1)
move = Move.play(point)
else:
move = Move.pass_turn()
if first_move_done and point is not None:
encode = True
# # Data only for debute Nail
# if self.count_stones_debute is not None and \
# self.count_stones_middle is None and \
# self.count_stones_end is None and \
# board_ext.count_stones() > self.count_stones_debute:
# encode = False
# # Data for middle game Nail
# if self.count_stones_debute is not None and \
# self.count_stones_middle is not None and \
# self.count_stones_end is None and \
# self.board_ext.count_stones() <= self.count_stones_debute and board_ext.count_stones() > self.count_stones_middle:
# encode = False
#
# # Data for end
# if self.count_stones_middle is not None and \
# self.count_stones_end is not None and \
# self.board_ext.count_stones() <= self.count_stones_middle and board_ext.count_stones() > self.count_stones_end:
# encode = False
if encode == True: #Nail
if self.encoder.name()[:2] == 'my':
features[counter] = self.encoder.encode(
game_state, board_ext) #Nail
else:
features[counter] = self.encoder.encode(
game_state)
labels[counter] = self.encoder.encode_point(point)
counter += 1
game_state = game_state.apply_move(move)
if self.encoder.name()[:2] == 'my':
board_ext.place_stone_ext(game_state.board, color,
point) # Inserted Nail
# Nail
first_move_done = True
feature_file_base = self.data_dir + '/' + data_file_name + '_features_%d'
label_file_base = self.data_dir + '/' + data_file_name + '_labels_%d'
chunk = 0 # Due to files with large content, split up after chunksize
chunksize = 1024
start_time_all = time.time()
while features.shape[0] >= chunksize:
start_time = time.time()
feature_file = feature_file_base % chunk
label_file = label_file_base % chunk
chunk += 1
current_features, features = features[:chunksize], features[
chunksize:]
current_labels, labels = labels[:chunksize], labels[chunksize:]
np.save(feature_file, current_features)
np.save(label_file, current_labels)
# Inserted Nail
print("Chunk = ", chunk, " File for training Current_features: ",
feature_file)
print("Time per one file = ",
(time.time() - start_time_all) / 1000, ' seconds')
print('Files preparation with proccess_zip is over\n')
print('Full Time = ', (time.time() - start_time_all) / 1000,
' seconds')
print("End chunk = ", chunk)
def decode_move_index(self, index):
if index == self.board_size * self.board_size:
return Move.pass_turn()
row = index // self.board_size
col = index % self.board_size
return Move.play(Point(row=row + 1, col=col + 1))
def test_capture_two_stones(self):
board = Board(19, 19)
board.place_stone(Player.black, Point(2, 2))
board.place_stone(Player.black, Point(2, 3))
board.place_stone(Player.white, Point(1, 2))
board.place_stone(Player.white, Point(1, 3))
self.assertEqual(Player.black, board.get(Point(2, 2)))
self.assertEqual(Player.black, board.get(Point(2, 3)))
board.place_stone(Player.white, Point(3, 2))
board.place_stone(Player.white, Point(3, 3))
self.assertEqual(Player.black, board.get(Point(2, 2)))
self.assertEqual(Player.black, board.get(Point(2, 3)))
board.place_stone(Player.white, Point(2, 1))
board.place_stone(Player.white, Point(2, 4))
self.assertIsNone(board.get(Point(2, 2)))
self.assertIsNone(board.get(Point(2, 3)))
def process_zip(self, zip_file_name, data_file_name, game_list):
# total number of moves in all games in zip file
tar_file = self.unzip_data(zip_file_name)
zip_file = tarfile.open(self.data_dir + '/' + tar_file)
name_list = zip_file.getnames()
total_examples = self.num_total_examples(zip_file, game_list, name_list)
new_game_list = game_list.copy()
shape = self.encoder.shape()
# Changed code to prevent too big features in memory
# Finish implementation of process_zip
feature_file_base = self.data_dir + '/' + data_file_name + '_features_%d'
label_file_base = self.data_dir + '/' + data_file_name + '_labels_%d'
chunk = 0
examples_used = 1024
iters = math.ceil(total_examples / examples_used)
for z in range(iters):
# <2>
#feature_shape = np.insert(shape, 0, np.asarray([examples_used]))
# features = np.zeros(feature_shape)
# labels = np.zeros((examples_used,))
# counter = 0
for index in new_game_list[:examples_used]:
features = []
labels = []
# reads the SGF contents as a string
name = name_list[index + 1]
if not name.endswith('.sgf'):
raise ValueError(name + ' is not a valid sgf')
sgf_content = zip_file.extractfile(name).read()
sgf = SgfGame.from_string(sgf_content)
# apply handicap stones
game_state, first_move_done = self.get_handicap(sgf)
# iterates through all moves
for item in sgf.main_sequence_iter():
z = np.zeros(shape)
color, move_tuple = item.get_move()
point = None
if color is not None:
# read the coordinate of the stone to be played
if move_tuple is not None:
row, col = move_tuple
point = Point(row + 1, col + 1)
move = Move.play(point)
else:
# or do nothing
move = Move.pass_turn()
if first_move_done and point is not None:
# Encode the current games state as features
# features[counter] = self.encoder.encode(game_state)
z = self.encoder.encode(game_state)
features.append(z)
# Encode the next move as label
# labels[counter] = self.encoder.encode_point(point)
labels.append(self.encoder.encode_point(point))
# counter += 1
# Apply the move and move on to the next one.
game_state = game_state.apply_move(move)
first_move_done = True
new_game_list = new_game_list[examples_used:]
feature_file = feature_file_base % chunk
label_file = label_file_base % chunk
chunk += 1
# current_features, features = features[:examples_used], features[examples_used:]
# current_labels, labels = labels[:examples_used], labels[examples_used:]
# np.save(feature_file, current_features)
# np.save(label_file, current_labels)
features = np.array(features)
labels = np.array(labels)
np.save(feature_file, features)
np.save(label_file, labels)
#
"""
def gtp_position_to_coords(gtp_position):
col_str, row_str = gtp_position[0], gtp_position[1:]
point = Point(int(row_str), COLS.find(col_str.upper()) + 1)
return Move(point)
from dlgo.gosgf import Sgf_game
from dlgo.goboard import GameState, Move
from dlgo.gotypes import Point
from dlgo.utils import print_board
from time import sleep
sgf_example = "(;GM[1]FF[4]SZ[9];W[ef];B[ff];W[df];B[fe];W[fc];B[ec];W[gd];B[fb])"
sgf_game = Sgf_game.from_string(sgf_example)
game_state = GameState.new_game(19)
for item in sgf_game.main_sequence_iter():
color, move_tuple = item.get_move()
if color is not None and move_tuple is not None:
row, col = move_tuple
point = Point(row + 1, col + 1)
move = Move.play(point)
game_state = game_state.apply_move(move)
print_board(game_state.board)
sleep(0.3)
def process_zip(self, zip_file_name, data_file_name, game_list):
tar_file = self.unzip_data(zip_file_name)
zip_file = tarfile.open(self.data_dir + '/' + tar_file)
name_list = zip_file.getnames()
total_examples = self.num_total_examples(zip_file, game_list, name_list) # <1>
shape = self.encoder.shape() # <2>
feature_shape = np.insert(shape, 0, np.asarray([total_examples]))
features = np.zeros(feature_shape)
labels = np.zeros((total_examples,))
counter = 0
for index in game_list:
name = name_list[index + 1]
if not name.endswith('.sgf'):
raise ValueError(name + ' is not a valid sgf')
sgf_content = zip_file.extractfile(name).read()
sgf = Sgf_game.from_string(sgf_content) # <3>
game_state, first_move_done = self.get_handicap(sgf) # <4>
for item in sgf.main_sequence_iter(): # <5>
color, move_tuple = item.get_move()
point = None
if color is not None:
if move_tuple is not None: # <6>
row, col = move_tuple
point = Point(row + 1, col + 1)
move = Move.play(point)
else:
move = Move.pass_turn() # <7>
if first_move_done and point is not None:
features[counter] = self.encoder.encode(game_state) # <8>
labels[counter] = self.encoder.encode_point(point) # <9>
counter += 1
game_state = game_state.apply_move(move) # <10>
first_move_done = True
# <1> Determine the total number of moves in all games in this zip file.
# <2> Infer the shape of features and labels from the encoder we use.
# <3> Read the SGF content as string, after extracting the zip file.
# <4> Infer the initial game state by applying all handicap stones.
# <5> Iterate over all moves in the SGF file.
# <6> Read the coordinates of the stone to be played...
# <7> ... or pass, if there is none.
# <8> We encode the current game state as features...
# <9> ... and the next move as label for the features.
# <10> Afterwards the move is applied to the board and we proceed with the next one.
# end::read_sgf_files[]
# tag::store_features_and_labels[]
feature_file_base = self.data_dir + '/' + data_file_name + '_features_%d'
label_file_base = self.data_dir + '/' + data_file_name + '_labels_%d'
chunk = 0 # Due to files with large content, split up after chunksize
chunksize = 1024
while features.shape[0] >= chunksize: # <1>
feature_file = feature_file_base % chunk
label_file = label_file_base % chunk
chunk += 1
current_features, features = features[:chunksize], features[chunksize:]
current_labels, labels = labels[:chunksize], labels[chunksize:] # <2>
np.save(feature_file, current_features)
np.save(label_file, current_labels) # <3>
def process_zip(self, zip_file_name, data_file_name, game_list):
# total number of moves in all games in zip file
tar_file = self.unzip_data(zip_file_name)
zip_file = tarfile.open(self.data_dir + '/' + tar_file)
name_list = zip_file.getnames()
total_examples = self.num_total_examples(zip_file, game_list,
name_list)
# infers shape of features and labels from the encoder
shape = self.encoder.shape() # <2>
feature_shape = np.insert(shape, 0, np.asarray([total_examples]))
features = np.zeros(feature_shape)
labels = np.zeros((total_examples, ))
counter = 0
for index in game_list:
# reads the SGF contents as a string
name = name_list[index + 1]
if not name.endswith('.sgf'):
raise ValueError(name + ' is not a valid sgf')
sgf_content = zip_file.extractfile(name).read()
sgf = SgfGame.from_string(sgf_content)
# apply handicap stones
game_state, first_move_done = self.get_handicap(sgf)
# iterates through all moves
for item in sgf.main_sequence_iter():
color, move_tuple = item.get_move()
point = None
if color is not None:
# read the coordinate of the stone to be played
if move_tuple is not None:
row, col = move_tuple
point = Point(row + 1, col + 1)
move = Move.play(point)
else:
# or do nothing
move = Move.pass_turn()
if first_move_done and point is not None:
# Encode the current games state as features
features[counter] = self.encoder.encode(game_state)
# Encode the next move as label
labels[counter] = self.encoder.encode_point(point)
counter += 1
# Apply the move and move on to the next one.
game_state = game_state.apply_move(move)
first_move_done = True
# Finish implementation of process_zip
feature_file_base = self.data_dir + '/' + data_file_name + '_features_%d'
label_file_base = self.data_dir + '/' + data_file_name + '_labels_%d'
chunk = 0 # Due to files with large content, split up after chunksize
chunksize = 1024
# author's code which doesn't do what he thinks it does
# # Process features and labels in chunks of 1024
# while features.shape[0] >= chunksize:
# feature_file = feature_file_base % chunk
# label_file = label_file_base % chunk
# chunk += 1
# # break up chunk into another piece
# current_features, features = features[:chunksize], features[chunksize:]
# current_labels, labels = labels[:chunksize], labels[chunksize:]
# # store into a separate file
# np.save(feature_file, current_features)
# np.save(label_file, current_labels)
# fixed code:
while features.shape[0] > 0:
feature_file = feature_file_base % chunk
label_file = label_file_base % chunk
chunk += 1
current_features, features = features[:chunksize], features[
chunksize:]
current_labels, labels = labels[:chunksize], labels[chunksize:]
np.save(feature_file, current_features)
np.save(label_file, current_labels)
def encode(self, game_state):
board_tensor = np.zeros(
(self.num_planes, self.board_height, self.board_width))
for r in range(self.board_height):
for c in range(self.board_width):
point = Point(row=r + 1, col=c + 1)
go_string = game_state.board.get_go_string(point)
if go_string and go_string.color == game_state.next_player:
board_tensor[offset("stone_color")][r][c] = 1
elif go_string and go_string.color == game_state.next_player.other:
board_tensor[offset("stone_color") + 1][r][c] = 1
else:
board_tensor[offset("stone_color") + 2][r][c] = 1
board_tensor[offset("ones")] = self.ones()
board_tensor[offset("zeros")] = self.zeros()
if not is_point_an_eye(game_state.board, point,
game_state.next_player):
board_tensor[offset("sensibleness")][r][c] = 1
ages = min(game_state.board.move_ages.get(r, c), 8)
if ages > 0:
print(ages)
board_tensor[int(offset("turns_since") + ages)][r][c] = 1
if game_state.board.get_go_string(point):
liberties = min(
game_state.board.get_go_string(point).num_liberties, 8)
board_tensor[offset("liberties") + liberties][r][c] = 1
move = Move(point)
if game_state.is_valid_move(move):
new_state = game_state.apply_move(move)
liberties = min(
new_state.board.get_go_string(point).num_liberties, 8)
board_tensor[offset("liberties_after") +
liberties][r][c] = 1
adjacent_strings = [
game_state.board.get_go_string(nb)
for nb in point.neighbors()
]
capture_count = 0
for go_string in adjacent_strings:
other_player = game_state.next_player.other
if go_string and go_string.num_liberties == 1 and go_string.color == other_player:
capture_count += len(go_string.stones)
capture_count = min(capture_count, 8)
board_tensor[offset("capture_size") +
capture_count][r][c] = 1
if go_string and go_string.num_liberties == 1:
go_string = game_state.board.get_go_string(point)
if go_string:
num_atari_stones = min(len(go_string.stones), 8)
board_tensor[offset("self_atari_size") +
num_atari_stones][r][c] = 1
if is_ladder_capture(game_state, point):
board_tensor[offset("ladder_capture")][r][c] = 1
if is_ladder_escape(game_state, point):
board_tensor[offset("ladder_escape")][r][c] = 1
if self.use_player_plane:
if game_state.next_player == Player.black:
board_tensor[offset("ones")] = self.ones()
else:
board_tensor[offset("zeros")] = self.zeros()
return board_tensor
from dlgo.gotypes import Player, Point
__all__ = ['HASH_CODE', 'EMPTY_BOARD']
HASH_CODE = {
(Point(row=1, col=1), Player.black): 6030943248300589574,
(Point(row=1, col=1), Player.white): 519642562745420640,
(Point(row=1, col=2), Player.black): 1676197564120249811,
(Point(row=1, col=2), Player.white): 3944757074220668448,
(Point(row=1, col=3), Player.black): 8672673169544686616,
(Point(row=1, col=3), Player.white): 5334518338003060721,
(Point(row=1, col=4), Player.black): 7138085118581435464,
(Point(row=1, col=4), Player.white): 4937890594369391536,
(Point(row=1, col=5), Player.black): 3031191868802541447,
(Point(row=1, col=5), Player.white): 119403534887610932,
(Point(row=1, col=6), Player.black): 290634728588546764,
(Point(row=1, col=6), Player.white): 6278747656778642845,
(Point(row=1, col=7), Player.black): 2206029329956200076,
(Point(row=1, col=7), Player.white): 3259542393520883112,
(Point(row=1, col=8), Player.black): 1847725693653552573,
(Point(row=1, col=8), Player.white): 2032931687138161754,
(Point(row=1, col=9), Player.black): 4560094825608761235,
(Point(row=1, col=9), Player.white): 8362496076491135448,
(Point(row=1, col=10), Player.black): 5519001134776394705,
(Point(row=1, col=10), Player.white): 5751651675906871892,
(Point(row=1, col=11), Player.black): 1900323695973797433,
(Point(row=1, col=11), Player.white): 2469287410892680640,
(Point(row=1, col=12), Player.black): 5199261569568851135,
(Point(row=1, col=12), Player.white): 2809915086007927513,
(Point(row=1, col=13), Player.black): 7690248179547487348,
(Point(row=1, col=13), Player.white): 7823808002424911219,
(Point(row=1, col=14), Player.black): 6860130047150569496,
def test_remove_liberties(self):
board = Board(5, 5)
board.place_stone(Player.black, Point(3, 3))
board.place_stone(Player.white, Point(2, 2))
white_string = board.get_go_string(Point(2, 2))
six.assertCountEqual(
self,
[Point(2, 3), Point(2, 1), Point(1, 2), Point(3, 2)],
white_string.liberties)
board.place_stone(Player.black, Point(3, 2))
white_string = board.get_go_string(Point(2, 2))
six.assertCountEqual(
self,
[Point(2, 3), Point(2, 1), Point(1, 2)],
white_string.liberties)
import random
from dlgo.gotypes import Player, Point
def to_python(player_state):
if player_state is None:
return 'None'
if player_state == Player.black:
return Player.black
return Player.white
MAX63 = 0x7fffffffffffffff
table = {}
empty_board = 0
for row in range(1,20):
for col in range(1,20):
for state in (Player.black, Player.white):
code = random.randint(0,MAX63)
table[Point(row,col), state] = code
print('from .gotypes imoprt Player, Point')
print('')
print("_all__ = ['HASH_CODE', 'EMPTY_BOARD']")
print('')
print('HASH_CODE = {')
for (pt, state), hash_code in table.items():
print(' (%r, %s): %r,' % (pt, to_python(state), hash_code))
print('}')
print('')
print('EMPTY_BOARD = %d' % (empty_board,))
def to_python(player_state):
if player_state is None:
return 'None'
if player_state == Player.black:
return Player.black
return Player.white
MAX63 = 0x7fffffffffffffff
tabel = {}
empty_board = 0
for row in range(1, 20):
for col in range(1, 20):
for state in (Player.black, Player.white):
code = random.randint(0, MAX63)
tabel[Point(row, col), state] = code
print('from .gotypes import Player, Point')
print('')
print("__all__ = ['HASH_CODE', 'EMOTY_BOARD']")
print('')
print('HASH_CODE = {')
for (pt, state), hash_code in tabel.items():
print(' (%r, %s): %r,' % (pt, to_python(state), hash_code))
print('}')
print('')
print('EMPTY_BOARD = %d' % (empty_board, ))