def test_make_dataset_from_sgf(self): with tempfile.NamedTemporaryFile() as sgf_file, \ tempfile.NamedTemporaryFile() as record_file: sgf_file.write(TEST_SGF.encode('utf8')) sgf_file.seek(0) preprocessing.make_dataset_from_sgf( utils_test.BOARD_SIZE, sgf_file.name, record_file.name) recovered_data = self.extract_data(record_file.name) start_pos = go.Position(utils_test.BOARD_SIZE) first_move = coords.from_sgf('fd') next_pos = start_pos.play_move(first_move) second_move = coords.from_sgf('cf') expected_data = [ ( features.extract_features(utils_test.BOARD_SIZE, start_pos), preprocessing._one_hot(utils_test.BOARD_SIZE, coords.to_flat( utils_test.BOARD_SIZE, first_move)), -1 ), ( features.extract_features(utils_test.BOARD_SIZE, next_pos), preprocessing._one_hot(utils_test.BOARD_SIZE, coords.to_flat( utils_test.BOARD_SIZE, second_move)), -1 ) ] self.assertEqualData(expected_data, recovered_data)
def test_pass(self): self.assertEqual(None, coords.from_sgf('')) self.assertEqual(None, coords.from_sgf('tt')) self.assertEqual(None, coords.from_flat(81)) self.assertEqual(None, coords.from_gtp('pass')) self.assertEqual(None, coords.from_gtp('PASS')) self.assertEqual('', coords.to_sgf(None)) self.assertEqual(81, coords.to_flat(None)) self.assertEqual('pass', coords.to_gtp(None))
def handle_node(pos, node): 'A node can either add B+W stones, play as B, or play as W.' props = node.properties black_stones_added = [coords.from_sgf(c) for c in props.get('AB', [])] white_stones_added = [coords.from_sgf(c) for c in props.get('AW', [])] if black_stones_added or white_stones_added: return add_stones(pos, black_stones_added, white_stones_added) # If B/W props are not present, then there is no move. But if it is present and equal to the empty string, then the move was a pass. elif 'B' in props: black_move = coords.from_sgf(props.get('B', [''])[0]) return pos.play_move(black_move, color=go.BLACK) elif 'W' in props: white_move = coords.from_sgf(props.get('W', [''])[0]) return pos.play_move(white_move, color=go.WHITE) else: return pos
def test_pass(self): self.assertEqual(None, coords.from_sgf('')) self.assertEqual(None, coords.from_flat(81)) self.assertEqual(None, coords.from_kgs('pass')) self.assertEqual('', coords.to_sgf(None)) self.assertEqual(81, coords.to_flat(None)) self.assertEqual('pass', coords.to_kgs(None))
def test_upperleft(self): self.assertEqual((0, 0), coords.from_sgf('aa')) self.assertEqual((0, 0), coords.from_flat(0)) self.assertEqual((0, 0), coords.from_kgs('A9')) self.assertEqual('aa', coords.to_sgf((0, 0))) self.assertEqual(0, coords.to_flat((0, 0))) self.assertEqual('A9', coords.to_kgs((0, 0)))
def test_topleft(self): self.assertEqual((0, 8), coords.from_sgf('ia')) self.assertEqual((0, 8), coords.from_flat(8)) self.assertEqual((0, 8), coords.from_kgs('J9')) self.assertEqual('ia', coords.to_sgf((0, 8))) self.assertEqual(8, coords.to_flat((0, 8))) self.assertEqual('J9', coords.to_kgs((0, 8)))
def test_pass(self): self.assertEqual(coords.from_sgf(''), None) self.assertEqual(coords.from_flat(81), None) self.assertEqual(coords.from_kgs('pass'), None) self.assertEqual(coords.to_sgf(None), '') self.assertEqual(coords.to_flat(None), 81) self.assertEqual(coords.to_kgs(None), 'pass')
def test_upperleft(self): self.assertEqual(coords.from_sgf('aa'), (0, 0)) self.assertEqual(coords.from_flat(0), (0, 0)) self.assertEqual(coords.from_kgs('A9'), (0, 0)) self.assertEqual(coords.to_sgf((0, 0)), 'aa') self.assertEqual(coords.to_flat((0, 0)), 0) self.assertEqual(coords.to_kgs((0, 0)), 'A9')
def handle_node(board_size, pos, node): """A node can either add B+W stones, play as B, or play as W.""" props = node.properties black_stones_added = [coords.from_sgf(c) for c in props.get('AB', [])] white_stones_added = [coords.from_sgf(c) for c in props.get('AW', [])] if black_stones_added or white_stones_added: return add_stones(board_size, pos, black_stones_added, white_stones_added) # If B/W props are not present, then there is no move. But if it is present # and equal to the empty string, then the move was a pass. elif 'B' in props: black_move = coords.from_sgf(props.get('B', [''])[0]) return pos.play_move(black_move, color=go.BLACK) elif 'W' in props: white_move = coords.from_sgf(props.get('W', [''])[0]) return pos.play_move(white_move, color=go.WHITE) else: return pos
def test_topleft(self): self.assertEqual(coords.from_sgf('ia'), (0, 8)) self.assertEqual(coords.from_flat(8), (0, 8)) self.assertEqual(coords.from_kgs('J9'), (0, 8)) self.assertEqual(coords.to_sgf((0, 8)), 'ia') self.assertEqual(coords.to_flat((0, 8)), 8) self.assertEqual(coords.to_kgs((0, 8)), 'J9')
def test_parsing_9x9(self): self.assertEqual(coords.from_sgf('aa'), (0, 0)) self.assertEqual(coords.from_sgf('ac'), (2, 0)) self.assertEqual(coords.from_sgf('ca'), (0, 2)) self.assertEqual(coords.from_sgf(''), None) self.assertEqual(coords.to_sgf(None), '') self.assertEqual('aa', coords.to_sgf(coords.from_sgf('aa'))) self.assertEqual('sa', coords.to_sgf(coords.from_sgf('sa'))) self.assertEqual((1, 17), coords.from_sgf(coords.to_sgf((1, 17)))) self.assertEqual(coords.from_kgs(utils_test.BOARD_SIZE, 'A1'), (8, 0)) self.assertEqual(coords.from_kgs(utils_test.BOARD_SIZE, 'A9'), (0, 0)) self.assertEqual(coords.from_kgs(utils_test.BOARD_SIZE, 'C2'), (7, 2)) self.assertEqual(coords.from_kgs(utils_test.BOARD_SIZE, 'J2'), (7, 8)) self.assertEqual(coords.from_pygtp(utils_test.BOARD_SIZE, (1, 1)), (8, 0)) self.assertEqual(coords.from_pygtp(utils_test.BOARD_SIZE, (1, 9)), (0, 0)) self.assertEqual(coords.from_pygtp(utils_test.BOARD_SIZE, (3, 2)), (7, 2)) self.assertEqual(coords.to_pygtp(utils_test.BOARD_SIZE, (8, 0)), (1, 1)) self.assertEqual(coords.to_pygtp(utils_test.BOARD_SIZE, (0, 0)), (1, 9)) self.assertEqual(coords.to_pygtp(utils_test.BOARD_SIZE, (7, 2)), (3, 2)) self.assertEqual(coords.to_kgs(utils_test.BOARD_SIZE, (0, 8)), 'J9') self.assertEqual(coords.to_kgs(utils_test.BOARD_SIZE, (8, 0)), 'A1')
def test_parsing_9x9(self): self.assertEqual(coords.from_sgf('aa'), (0, 0)) self.assertEqual(coords.from_sgf('ac'), (2, 0)) self.assertEqual(coords.from_sgf('ca'), (0, 2)) self.assertEqual(coords.from_sgf(''), None) self.assertEqual(coords.to_sgf(None), '') self.assertEqual( 'aa', coords.to_sgf(coords.from_sgf('aa'))) self.assertEqual( 'sa', coords.to_sgf(coords.from_sgf('sa'))) self.assertEqual( (1, 17), coords.from_sgf(coords.to_sgf((1, 17)))) self.assertEqual(coords.from_kgs('A1'), (8, 0)) self.assertEqual(coords.from_kgs('A9'), (0, 0)) self.assertEqual(coords.from_kgs('C2'), (7, 2)) self.assertEqual(coords.from_kgs('J2'), (7, 8)) self.assertEqual(coords.from_pygtp((1, 1)), (8, 0)) self.assertEqual(coords.from_pygtp((1, 9)), (0, 0)) self.assertEqual(coords.from_pygtp((3, 2)), (7, 2)) self.assertEqual(coords.to_pygtp((8, 0)), (1, 1)) self.assertEqual(coords.to_pygtp((0, 0)), (1, 9)) self.assertEqual(coords.to_pygtp((7, 2)), (3, 2)) self.assertEqual(coords.to_kgs((0, 8)), 'J9') self.assertEqual(coords.to_kgs((8, 0)), 'A1')
def test_topleft(self): self.assertEqual(coords.from_sgf('ia'), (0, 8)) self.assertEqual(coords.from_flat(utils_test.BOARD_SIZE, 8), (0, 8)) self.assertEqual(coords.from_kgs(utils_test.BOARD_SIZE, 'J9'), (0, 8)) self.assertEqual(coords.from_pygtp(utils_test.BOARD_SIZE, (9, 9)), (0, 8)) self.assertEqual(coords.to_sgf((0, 8)), 'ia') self.assertEqual(coords.to_flat(utils_test.BOARD_SIZE, (0, 8)), 8) self.assertEqual(coords.to_kgs(utils_test.BOARD_SIZE, (0, 8)), 'J9') self.assertEqual(coords.to_pygtp(utils_test.BOARD_SIZE, (0, 8)), (9, 9))
def test_pass(self): self.assertEqual(coords.from_sgf(''), None) self.assertEqual(coords.from_flat(81), None) self.assertEqual(coords.from_kgs('pass'), None) self.assertEqual(coords.from_pygtp((0, 0)), None) self.assertEqual(coords.to_sgf(None), '') self.assertEqual(coords.to_flat(None), 81) self.assertEqual(coords.to_kgs(None), 'pass') self.assertEqual(coords.to_pygtp(None), (0, 0))
def test_topleft(self): self.assertEqual(coords.from_sgf('ia'), (0, 8)) self.assertEqual(coords.from_flat(8), (0, 8)) self.assertEqual(coords.from_kgs('J9'), (0, 8)) self.assertEqual(coords.from_pygtp((9, 9)), (0, 8)) self.assertEqual(coords.to_sgf((0, 8)), 'ia') self.assertEqual(coords.to_flat((0, 8)), 8) self.assertEqual(coords.to_kgs((0, 8)), 'J9') self.assertEqual(coords.to_pygtp((0, 8)), (9, 9))
def test_upperleft(self): self.assertEqual(coords.from_sgf('aa'), (0, 0)) self.assertEqual(coords.from_flat(0), (0, 0)) self.assertEqual(coords.from_kgs('A9'), (0, 0)) self.assertEqual(coords.from_pygtp((1, 9)), (0, 0)) self.assertEqual(coords.to_sgf((0, 0)), 'aa') self.assertEqual(coords.to_flat((0, 0)), 0) self.assertEqual(coords.to_kgs((0, 0)), 'A9') self.assertEqual(coords.to_pygtp((0, 0)), (1, 9))
def test_pass(self): self.assertEqual(coords.from_sgf(''), None) self.assertEqual(coords.from_flat(utils_test.BOARD_SIZE, 81), None) self.assertEqual(coords.from_kgs(utils_test.BOARD_SIZE, 'pass'), None) self.assertEqual(coords.from_pygtp(utils_test.BOARD_SIZE, (0, 0)), None) self.assertEqual(coords.to_sgf(None), '') self.assertEqual(coords.to_flat(utils_test.BOARD_SIZE, None), 81) self.assertEqual(coords.to_kgs(utils_test.BOARD_SIZE, None), 'pass') self.assertEqual(coords.to_pygtp(utils_test.BOARD_SIZE, None), (0, 0))
def test_upperleft(self): self.assertEqual(coords.from_sgf('aa'), (0, 0)) self.assertEqual(coords.from_flat(utils_test.BOARD_SIZE, 0), (0, 0)) self.assertEqual(coords.from_kgs(utils_test.BOARD_SIZE, 'A9'), (0, 0)) self.assertEqual(coords.from_pygtp(utils_test.BOARD_SIZE, (1, 9)), (0, 0)) self.assertEqual(coords.to_sgf((0, 0)), 'aa') self.assertEqual(coords.to_flat(utils_test.BOARD_SIZE, (0, 0)), 0) self.assertEqual(coords.to_kgs(utils_test.BOARD_SIZE, (0, 0)), 'A9') self.assertEqual(coords.to_pygtp(utils_test.BOARD_SIZE, (0, 0)), (1, 9))
def test_make_dataset_from_sgf(self): with tempfile.NamedTemporaryFile() as sgf_file, \ tempfile.NamedTemporaryFile() as record_file: sgf_file.write(TEST_SGF.encode('utf8')) sgf_file.seek(0) preprocessing.make_dataset_from_sgf(sgf_file.name, record_file.name) recovered_data = self.extract_data(record_file.name) start_pos = go.Position() first_move = coords.from_sgf('fd') next_pos = start_pos.play_move(first_move) second_move = coords.from_sgf('cf') expected_data = [ (features.extract_features(start_pos), preprocessing._one_hot(coords.to_flat(first_move)), -1), (features.extract_features(next_pos), preprocessing._one_hot(coords.to_flat(second_move)), -1) ] self.assertEqualData(expected_data, recovered_data)
def test_pass(self): self.assertEqual(coords.from_sgf(''), None) self.assertEqual(coords.from_flat(utils_test.BOARD_SIZE, 81), None) self.assertEqual(coords.from_kgs(utils_test.BOARD_SIZE, 'pass'), None) self.assertEqual(coords.from_pygtp(utils_test.BOARD_SIZE, (0, 0)), None) self.assertEqual(coords.to_sgf(None), '') self.assertEqual(coords.to_flat(utils_test.BOARD_SIZE, None), 81) self.assertEqual(coords.to_kgs(utils_test.BOARD_SIZE, None), 'pass') self.assertEqual(coords.to_pygtp(utils_test.BOARD_SIZE, None), (0, 0))
def test_topleft(self): self.assertEqual(coords.from_sgf('ia'), (0, 8)) self.assertEqual(coords.from_flat(utils_test.BOARD_SIZE, 8), (0, 8)) self.assertEqual(coords.from_kgs(utils_test.BOARD_SIZE, 'J9'), (0, 8)) self.assertEqual(coords.from_pygtp(utils_test.BOARD_SIZE, (9, 9)), (0, 8)) self.assertEqual(coords.to_sgf((0, 8)), 'ia') self.assertEqual(coords.to_flat(utils_test.BOARD_SIZE, (0, 8)), 8) self.assertEqual(coords.to_kgs(utils_test.BOARD_SIZE, (0, 8)), 'J9') self.assertEqual(coords.to_pygtp(utils_test.BOARD_SIZE, (0, 8)), (9, 9))
def test_upperleft(self): self.assertEqual(coords.from_sgf('aa'), (0, 0)) self.assertEqual(coords.from_flat(utils_test.BOARD_SIZE, 0), (0, 0)) self.assertEqual(coords.from_kgs(utils_test.BOARD_SIZE, 'A9'), (0, 0)) self.assertEqual(coords.from_pygtp(utils_test.BOARD_SIZE, (1, 9)), (0, 0)) self.assertEqual(coords.to_sgf((0, 0)), 'aa') self.assertEqual(coords.to_flat(utils_test.BOARD_SIZE, (0, 0)), 0) self.assertEqual(coords.to_kgs(utils_test.BOARD_SIZE, (0, 0)), 'A9') self.assertEqual(coords.to_pygtp(utils_test.BOARD_SIZE, (0, 0)), (1, 9))
def test_parsing_9x9(self): self.assertEqual((0, 0), coords.from_sgf('aa')) self.assertEqual((2, 0), coords.from_sgf('ac')) self.assertEqual((0, 2), coords.from_sgf('ca')) self.assertEqual(None, coords.from_sgf('')) self.assertEqual('', coords.to_sgf(None)) self.assertEqual('aa', coords.to_sgf(coords.from_sgf('aa'))) self.assertEqual('sa', coords.to_sgf(coords.from_sgf('sa'))) self.assertEqual((1, 17), coords.from_sgf(coords.to_sgf((1, 17)))) self.assertEqual((8, 0), coords.from_kgs('A1')) self.assertEqual((0, 0), coords.from_kgs('A9')) self.assertEqual((7, 2), coords.from_kgs('C2')) self.assertEqual((7, 8), coords.from_kgs('J2')) self.assertEqual('J9', coords.to_kgs((0, 8))) self.assertEqual('A1', coords.to_kgs((8, 0)))
def extract_move_data(root_node, worker_id, completed_time, board_size): current_node = root_node.next move_data = [] move_num = 1 while current_node is not None: props = current_node.properties if 'B' in props: to_play = 1 move_played = props['B'][0] elif 'W' in props: to_play = -1 move_played = props['W'][0] else: import pdb; pdb.set_trace() move_played = coords.to_flat(coords.from_sgf(move_played)) post_Q, debug_rows = parse_comment_node(props['C'][0]) policy_prior = [0] * (board_size * board_size + 1) policy_prior_orig = policy_prior[:] mcts_visit_counts = policy_prior[:] mcts_visit_counts_norm = policy_prior[:] for debug_row in debug_rows: move = debug_row.move policy_prior[move] = debug_row.prior policy_prior_orig[move] = debug_row.orig_prior mcts_visit_counts[move] = debug_row.N mcts_visit_counts_norm[move] = debug_row.soft_N move_data.append({ 'worker_id': worker_id, 'completed_time': completed_time, 'move_num': move_num, 'turn_to_play': to_play, 'move': move_played, 'move_kgs': coords.to_kgs(coords.from_flat(move_played)), 'prior_Q': None, 'post_Q': post_Q, 'policy_prior': policy_prior, 'policy_prior_orig': policy_prior_orig, 'mcts_visit_counts': mcts_visit_counts, 'mcts_visit_counts_norm': mcts_visit_counts_norm, }) move_num += 1 current_node = current_node.next return move_data
def get_next_move(node): props = node.next.properties if 'W' in props: return coords.from_sgf(props['W'][0]) else: return coords.from_sgf(props['B'][0])
def extract_move_data(root_node, worker_id, completed_time, board_size): current_node = root_node.next move_data = [] move_num = 1 while current_node is not None: props = current_node.properties if 'B' in props: to_play = 1 move_played = props['B'][0] elif 'W' in props: to_play = -1 move_played = props['W'][0] else: import pdb pdb.set_trace() move_played = coords.to_flat(coords.from_sgf(move_played)) post_Q, debug_rows = parse_comment_node(props['C'][0]) def get_row_data(debug_row): column_names = ["prior", "orig_prior", "N", "soft_N"] return [getattr(debug_row, field) for field in column_names] if FLAGS.only_top_move: assert len(debug_rows) <= 1 row_data = list(map(get_row_data, debug_rows)) else: row_data = [[0] * 4 for _ in range(board_size * board_size + 1)] for debug_row in debug_rows: move = debug_row.move row_data[move] = get_row_data(debug_row) policy_prior, policy_prior_orig, mcts_visits, mcts_visits_norm = \ zip(*row_data) move_data.append({ 'worker_id': worker_id, 'completed_time': completed_time, 'move_num': move_num, 'turn_to_play': to_play, 'move': move_played, 'move_kgs': coords.to_gtp(coords.from_flat(move_played)), 'prior_Q': None, 'post_Q': post_Q, 'policy_prior': policy_prior, 'policy_prior_orig': policy_prior_orig, 'mcts_visit_counts': mcts_visits, 'mcts_visit_counts_norm': mcts_visits_norm, }) move_num += 1 current_node = current_node.next return move_data
def get_next_move(node): props = node.next.properties if 'W' in props: return coords.from_sgf(props['W'][0]) else: return coords.from_sgf(props['B'][0])