def test_make_dataset_from_sgf(self):
with tempfile.NamedTemporaryFile() as sgf_file, \
tempfile.NamedTemporaryFile() as record_file:
sgf_file.write(TEST_SGF.encode('utf8'))
sgf_file.seek(0)
preprocessing.make_dataset_from_sgf(
utils_test.BOARD_SIZE, sgf_file.name, record_file.name)
recovered_data = self.extract_data(record_file.name)
start_pos = go.Position(utils_test.BOARD_SIZE)
first_move = coords.from_sgf('fd')
next_pos = start_pos.play_move(first_move)
second_move = coords.from_sgf('cf')
expected_data = [
(
features.extract_features(utils_test.BOARD_SIZE, start_pos),
preprocessing._one_hot(utils_test.BOARD_SIZE, coords.to_flat(
utils_test.BOARD_SIZE, first_move)), -1
),
(
features.extract_features(utils_test.BOARD_SIZE, next_pos),
preprocessing._one_hot(utils_test.BOARD_SIZE, coords.to_flat(
utils_test.BOARD_SIZE, second_move)), -1
)
]
self.assertEqualData(expected_data, recovered_data)
def test_pass(self):
self.assertEqual(None, coords.from_sgf(''))
self.assertEqual(None, coords.from_sgf('tt'))
self.assertEqual(None, coords.from_flat(81))
self.assertEqual(None, coords.from_gtp('pass'))
self.assertEqual(None, coords.from_gtp('PASS'))
self.assertEqual('', coords.to_sgf(None))
self.assertEqual(81, coords.to_flat(None))
self.assertEqual('pass', coords.to_gtp(None))
def handle_node(pos, node):
'A node can either add B+W stones, play as B, or play as W.'
props = node.properties
black_stones_added = [coords.from_sgf(c) for c in props.get('AB', [])]
white_stones_added = [coords.from_sgf(c) for c in props.get('AW', [])]
if black_stones_added or white_stones_added:
return add_stones(pos, black_stones_added, white_stones_added)
# If B/W props are not present, then there is no move. But if it is present and equal to the empty string, then the move was a pass.
elif 'B' in props:
black_move = coords.from_sgf(props.get('B', [''])[0])
return pos.play_move(black_move, color=go.BLACK)
elif 'W' in props:
white_move = coords.from_sgf(props.get('W', [''])[0])
return pos.play_move(white_move, color=go.WHITE)
else:
return pos
def test_pass(self):
self.assertEqual(None, coords.from_sgf(''))
self.assertEqual(None, coords.from_flat(81))
self.assertEqual(None, coords.from_kgs('pass'))
self.assertEqual('', coords.to_sgf(None))
self.assertEqual(81, coords.to_flat(None))
self.assertEqual('pass', coords.to_kgs(None))
def test_upperleft(self):
self.assertEqual((0, 0), coords.from_sgf('aa'))
self.assertEqual((0, 0), coords.from_flat(0))
self.assertEqual((0, 0), coords.from_kgs('A9'))
self.assertEqual('aa', coords.to_sgf((0, 0)))
self.assertEqual(0, coords.to_flat((0, 0)))
self.assertEqual('A9', coords.to_kgs((0, 0)))
def test_topleft(self):
self.assertEqual((0, 8), coords.from_sgf('ia'))
self.assertEqual((0, 8), coords.from_flat(8))
self.assertEqual((0, 8), coords.from_kgs('J9'))
self.assertEqual('ia', coords.to_sgf((0, 8)))
self.assertEqual(8, coords.to_flat((0, 8)))
self.assertEqual('J9', coords.to_kgs((0, 8)))
def test_pass(self):
self.assertEqual(coords.from_sgf(''), None)
self.assertEqual(coords.from_flat(81), None)
self.assertEqual(coords.from_kgs('pass'), None)
self.assertEqual(coords.to_sgf(None), '')
self.assertEqual(coords.to_flat(None), 81)
self.assertEqual(coords.to_kgs(None), 'pass')
def test_upperleft(self):
self.assertEqual(coords.from_sgf('aa'), (0, 0))
self.assertEqual(coords.from_flat(0), (0, 0))
self.assertEqual(coords.from_kgs('A9'), (0, 0))
self.assertEqual(coords.to_sgf((0, 0)), 'aa')
self.assertEqual(coords.to_flat((0, 0)), 0)
self.assertEqual(coords.to_kgs((0, 0)), 'A9')
def handle_node(board_size, pos, node):
"""A node can either add B+W stones, play as B, or play as W."""
props = node.properties
black_stones_added = [coords.from_sgf(c) for c in props.get('AB', [])]
white_stones_added = [coords.from_sgf(c) for c in props.get('AW', [])]
if black_stones_added or white_stones_added:
return add_stones(board_size, pos, black_stones_added, white_stones_added)
# If B/W props are not present, then there is no move. But if it is present
# and equal to the empty string, then the move was a pass.
elif 'B' in props:
black_move = coords.from_sgf(props.get('B', [''])[0])
return pos.play_move(black_move, color=go.BLACK)
elif 'W' in props:
white_move = coords.from_sgf(props.get('W', [''])[0])
return pos.play_move(white_move, color=go.WHITE)
else:
return pos
def test_topleft(self):
self.assertEqual(coords.from_sgf('ia'), (0, 8))
self.assertEqual(coords.from_flat(8), (0, 8))
self.assertEqual(coords.from_kgs('J9'), (0, 8))
self.assertEqual(coords.to_sgf((0, 8)), 'ia')
self.assertEqual(coords.to_flat((0, 8)), 8)
self.assertEqual(coords.to_kgs((0, 8)), 'J9')
def test_parsing_9x9(self):
self.assertEqual(coords.from_sgf('aa'), (0, 0))
self.assertEqual(coords.from_sgf('ac'), (2, 0))
self.assertEqual(coords.from_sgf('ca'), (0, 2))
self.assertEqual(coords.from_sgf(''), None)
self.assertEqual(coords.to_sgf(None), '')
self.assertEqual('aa', coords.to_sgf(coords.from_sgf('aa')))
self.assertEqual('sa', coords.to_sgf(coords.from_sgf('sa')))
self.assertEqual((1, 17), coords.from_sgf(coords.to_sgf((1, 17))))
self.assertEqual(coords.from_kgs(utils_test.BOARD_SIZE, 'A1'), (8, 0))
self.assertEqual(coords.from_kgs(utils_test.BOARD_SIZE, 'A9'), (0, 0))
self.assertEqual(coords.from_kgs(utils_test.BOARD_SIZE, 'C2'), (7, 2))
self.assertEqual(coords.from_kgs(utils_test.BOARD_SIZE, 'J2'), (7, 8))
self.assertEqual(coords.from_pygtp(utils_test.BOARD_SIZE, (1, 1)),
(8, 0))
self.assertEqual(coords.from_pygtp(utils_test.BOARD_SIZE, (1, 9)),
(0, 0))
self.assertEqual(coords.from_pygtp(utils_test.BOARD_SIZE, (3, 2)),
(7, 2))
self.assertEqual(coords.to_pygtp(utils_test.BOARD_SIZE, (8, 0)),
(1, 1))
self.assertEqual(coords.to_pygtp(utils_test.BOARD_SIZE, (0, 0)),
(1, 9))
self.assertEqual(coords.to_pygtp(utils_test.BOARD_SIZE, (7, 2)),
(3, 2))
self.assertEqual(coords.to_kgs(utils_test.BOARD_SIZE, (0, 8)), 'J9')
self.assertEqual(coords.to_kgs(utils_test.BOARD_SIZE, (8, 0)), 'A1')
def test_parsing_9x9(self):
self.assertEqual(coords.from_sgf('aa'), (0, 0))
self.assertEqual(coords.from_sgf('ac'), (2, 0))
self.assertEqual(coords.from_sgf('ca'), (0, 2))
self.assertEqual(coords.from_sgf(''), None)
self.assertEqual(coords.to_sgf(None), '')
self.assertEqual(
'aa',
coords.to_sgf(coords.from_sgf('aa')))
self.assertEqual(
'sa',
coords.to_sgf(coords.from_sgf('sa')))
self.assertEqual(
(1, 17),
coords.from_sgf(coords.to_sgf((1, 17))))
self.assertEqual(coords.from_kgs('A1'), (8, 0))
self.assertEqual(coords.from_kgs('A9'), (0, 0))
self.assertEqual(coords.from_kgs('C2'), (7, 2))
self.assertEqual(coords.from_kgs('J2'), (7, 8))
self.assertEqual(coords.from_pygtp((1, 1)), (8, 0))
self.assertEqual(coords.from_pygtp((1, 9)), (0, 0))
self.assertEqual(coords.from_pygtp((3, 2)), (7, 2))
self.assertEqual(coords.to_pygtp((8, 0)), (1, 1))
self.assertEqual(coords.to_pygtp((0, 0)), (1, 9))
self.assertEqual(coords.to_pygtp((7, 2)), (3, 2))
self.assertEqual(coords.to_kgs((0, 8)), 'J9')
self.assertEqual(coords.to_kgs((8, 0)), 'A1')
def test_topleft(self):
self.assertEqual(coords.from_sgf('ia'), (0, 8))
self.assertEqual(coords.from_flat(utils_test.BOARD_SIZE, 8), (0, 8))
self.assertEqual(coords.from_kgs(utils_test.BOARD_SIZE, 'J9'), (0, 8))
self.assertEqual(coords.from_pygtp(utils_test.BOARD_SIZE, (9, 9)), (0, 8))
self.assertEqual(coords.to_sgf((0, 8)), 'ia')
self.assertEqual(coords.to_flat(utils_test.BOARD_SIZE, (0, 8)), 8)
self.assertEqual(coords.to_kgs(utils_test.BOARD_SIZE, (0, 8)), 'J9')
self.assertEqual(coords.to_pygtp(utils_test.BOARD_SIZE, (0, 8)), (9, 9))
def test_pass(self):
self.assertEqual(coords.from_sgf(''), None)
self.assertEqual(coords.from_flat(81), None)
self.assertEqual(coords.from_kgs('pass'), None)
self.assertEqual(coords.from_pygtp((0, 0)), None)
self.assertEqual(coords.to_sgf(None), '')
self.assertEqual(coords.to_flat(None), 81)
self.assertEqual(coords.to_kgs(None), 'pass')
self.assertEqual(coords.to_pygtp(None), (0, 0))
def test_topleft(self):
self.assertEqual(coords.from_sgf('ia'), (0, 8))
self.assertEqual(coords.from_flat(8), (0, 8))
self.assertEqual(coords.from_kgs('J9'), (0, 8))
self.assertEqual(coords.from_pygtp((9, 9)), (0, 8))
self.assertEqual(coords.to_sgf((0, 8)), 'ia')
self.assertEqual(coords.to_flat((0, 8)), 8)
self.assertEqual(coords.to_kgs((0, 8)), 'J9')
self.assertEqual(coords.to_pygtp((0, 8)), (9, 9))
def test_upperleft(self):
self.assertEqual(coords.from_sgf('aa'), (0, 0))
self.assertEqual(coords.from_flat(0), (0, 0))
self.assertEqual(coords.from_kgs('A9'), (0, 0))
self.assertEqual(coords.from_pygtp((1, 9)), (0, 0))
self.assertEqual(coords.to_sgf((0, 0)), 'aa')
self.assertEqual(coords.to_flat((0, 0)), 0)
self.assertEqual(coords.to_kgs((0, 0)), 'A9')
self.assertEqual(coords.to_pygtp((0, 0)), (1, 9))
def test_pass(self):
self.assertEqual(coords.from_sgf(''), None)
self.assertEqual(coords.from_flat(utils_test.BOARD_SIZE, 81), None)
self.assertEqual(coords.from_kgs(utils_test.BOARD_SIZE, 'pass'), None)
self.assertEqual(coords.from_pygtp(utils_test.BOARD_SIZE, (0, 0)), None)
self.assertEqual(coords.to_sgf(None), '')
self.assertEqual(coords.to_flat(utils_test.BOARD_SIZE, None), 81)
self.assertEqual(coords.to_kgs(utils_test.BOARD_SIZE, None), 'pass')
self.assertEqual(coords.to_pygtp(utils_test.BOARD_SIZE, None), (0, 0))
def test_upperleft(self):
self.assertEqual(coords.from_sgf('aa'), (0, 0))
self.assertEqual(coords.from_flat(utils_test.BOARD_SIZE, 0), (0, 0))
self.assertEqual(coords.from_kgs(utils_test.BOARD_SIZE, 'A9'), (0, 0))
self.assertEqual(coords.from_pygtp(utils_test.BOARD_SIZE, (1, 9)), (0, 0))
self.assertEqual(coords.to_sgf((0, 0)), 'aa')
self.assertEqual(coords.to_flat(utils_test.BOARD_SIZE, (0, 0)), 0)
self.assertEqual(coords.to_kgs(utils_test.BOARD_SIZE, (0, 0)), 'A9')
self.assertEqual(coords.to_pygtp(utils_test.BOARD_SIZE, (0, 0)), (1, 9))
def test_make_dataset_from_sgf(self):
with tempfile.NamedTemporaryFile() as sgf_file, \
tempfile.NamedTemporaryFile() as record_file:
sgf_file.write(TEST_SGF.encode('utf8'))
sgf_file.seek(0)
preprocessing.make_dataset_from_sgf(sgf_file.name,
record_file.name)
recovered_data = self.extract_data(record_file.name)
start_pos = go.Position()
first_move = coords.from_sgf('fd')
next_pos = start_pos.play_move(first_move)
second_move = coords.from_sgf('cf')
expected_data = [
(features.extract_features(start_pos),
preprocessing._one_hot(coords.to_flat(first_move)), -1),
(features.extract_features(next_pos),
preprocessing._one_hot(coords.to_flat(second_move)), -1)
]
self.assertEqualData(expected_data, recovered_data)
def test_pass(self):
self.assertEqual(coords.from_sgf(''), None)
self.assertEqual(coords.from_flat(utils_test.BOARD_SIZE, 81), None)
self.assertEqual(coords.from_kgs(utils_test.BOARD_SIZE, 'pass'), None)
self.assertEqual(coords.from_pygtp(utils_test.BOARD_SIZE, (0, 0)),
None)
self.assertEqual(coords.to_sgf(None), '')
self.assertEqual(coords.to_flat(utils_test.BOARD_SIZE, None), 81)
self.assertEqual(coords.to_kgs(utils_test.BOARD_SIZE, None), 'pass')
self.assertEqual(coords.to_pygtp(utils_test.BOARD_SIZE, None), (0, 0))
def test_topleft(self):
self.assertEqual(coords.from_sgf('ia'), (0, 8))
self.assertEqual(coords.from_flat(utils_test.BOARD_SIZE, 8), (0, 8))
self.assertEqual(coords.from_kgs(utils_test.BOARD_SIZE, 'J9'), (0, 8))
self.assertEqual(coords.from_pygtp(utils_test.BOARD_SIZE, (9, 9)),
(0, 8))
self.assertEqual(coords.to_sgf((0, 8)), 'ia')
self.assertEqual(coords.to_flat(utils_test.BOARD_SIZE, (0, 8)), 8)
self.assertEqual(coords.to_kgs(utils_test.BOARD_SIZE, (0, 8)), 'J9')
self.assertEqual(coords.to_pygtp(utils_test.BOARD_SIZE, (0, 8)),
(9, 9))
def test_upperleft(self):
self.assertEqual(coords.from_sgf('aa'), (0, 0))
self.assertEqual(coords.from_flat(utils_test.BOARD_SIZE, 0), (0, 0))
self.assertEqual(coords.from_kgs(utils_test.BOARD_SIZE, 'A9'), (0, 0))
self.assertEqual(coords.from_pygtp(utils_test.BOARD_SIZE, (1, 9)),
(0, 0))
self.assertEqual(coords.to_sgf((0, 0)), 'aa')
self.assertEqual(coords.to_flat(utils_test.BOARD_SIZE, (0, 0)), 0)
self.assertEqual(coords.to_kgs(utils_test.BOARD_SIZE, (0, 0)), 'A9')
self.assertEqual(coords.to_pygtp(utils_test.BOARD_SIZE, (0, 0)),
(1, 9))
def test_parsing_9x9(self):
self.assertEqual((0, 0), coords.from_sgf('aa'))
self.assertEqual((2, 0), coords.from_sgf('ac'))
self.assertEqual((0, 2), coords.from_sgf('ca'))
self.assertEqual(None, coords.from_sgf(''))
self.assertEqual('', coords.to_sgf(None))
self.assertEqual('aa', coords.to_sgf(coords.from_sgf('aa')))
self.assertEqual('sa', coords.to_sgf(coords.from_sgf('sa')))
self.assertEqual((1, 17), coords.from_sgf(coords.to_sgf((1, 17))))
self.assertEqual((8, 0), coords.from_kgs('A1'))
self.assertEqual((0, 0), coords.from_kgs('A9'))
self.assertEqual((7, 2), coords.from_kgs('C2'))
self.assertEqual((7, 8), coords.from_kgs('J2'))
self.assertEqual('J9', coords.to_kgs((0, 8)))
self.assertEqual('A1', coords.to_kgs((8, 0)))
def extract_move_data(root_node, worker_id, completed_time, board_size):
current_node = root_node.next
move_data = []
move_num = 1
while current_node is not None:
props = current_node.properties
if 'B' in props:
to_play = 1
move_played = props['B'][0]
elif 'W' in props:
to_play = -1
move_played = props['W'][0]
else:
import pdb; pdb.set_trace()
move_played = coords.to_flat(coords.from_sgf(move_played))
post_Q, debug_rows = parse_comment_node(props['C'][0])
policy_prior = [0] * (board_size * board_size + 1)
policy_prior_orig = policy_prior[:]
mcts_visit_counts = policy_prior[:]
mcts_visit_counts_norm = policy_prior[:]
for debug_row in debug_rows:
move = debug_row.move
policy_prior[move] = debug_row.prior
policy_prior_orig[move] = debug_row.orig_prior
mcts_visit_counts[move] = debug_row.N
mcts_visit_counts_norm[move] = debug_row.soft_N
move_data.append({
'worker_id': worker_id,
'completed_time': completed_time,
'move_num': move_num,
'turn_to_play': to_play,
'move': move_played,
'move_kgs': coords.to_kgs(coords.from_flat(move_played)),
'prior_Q': None,
'post_Q': post_Q,
'policy_prior': policy_prior,
'policy_prior_orig': policy_prior_orig,
'mcts_visit_counts': mcts_visit_counts,
'mcts_visit_counts_norm': mcts_visit_counts_norm,
})
move_num += 1
current_node = current_node.next
return move_data
def get_next_move(node):
props = node.next.properties
if 'W' in props:
return coords.from_sgf(props['W'][0])
else:
return coords.from_sgf(props['B'][0])
def extract_move_data(root_node, worker_id, completed_time, board_size):
current_node = root_node.next
move_data = []
move_num = 1
while current_node is not None:
props = current_node.properties
if 'B' in props:
to_play = 1
move_played = props['B'][0]
elif 'W' in props:
to_play = -1
move_played = props['W'][0]
else:
import pdb
pdb.set_trace()
move_played = coords.to_flat(coords.from_sgf(move_played))
post_Q, debug_rows = parse_comment_node(props['C'][0])
def get_row_data(debug_row):
column_names = ["prior", "orig_prior", "N", "soft_N"]
return [getattr(debug_row, field) for field in column_names]
if FLAGS.only_top_move:
assert len(debug_rows) <= 1
row_data = list(map(get_row_data, debug_rows))
else:
row_data = [[0] * 4 for _ in range(board_size * board_size + 1)]
for debug_row in debug_rows:
move = debug_row.move
row_data[move] = get_row_data(debug_row)
policy_prior, policy_prior_orig, mcts_visits, mcts_visits_norm = \
zip(*row_data)
move_data.append({
'worker_id':
worker_id,
'completed_time':
completed_time,
'move_num':
move_num,
'turn_to_play':
to_play,
'move':
move_played,
'move_kgs':
coords.to_gtp(coords.from_flat(move_played)),
'prior_Q':
None,
'post_Q':
post_Q,
'policy_prior':
policy_prior,
'policy_prior_orig':
policy_prior_orig,
'mcts_visit_counts':
mcts_visits,
'mcts_visit_counts_norm':
mcts_visits_norm,
})
move_num += 1
current_node = current_node.next
return move_data
def get_next_move(node):
props = node.next.properties
if 'W' in props:
return coords.from_sgf(props['W'][0])
else:
return coords.from_sgf(props['B'][0])
