def evaluate(black_model: 'The path to the model to play black',
white_model: 'The path to the model to play white',
output_dir:
'Where to write the evaluation results' = 'data/evaluate/sgf',
readouts: 'How many readouts to make per move.' = 400,
games: 'the number of games to play' = 16,
verbose: 'How verbose the players should be (see selfplay)' = 1):
black_model = os.path.abspath(black_model)
white_model = os.path.abspath(white_model)
with timer("Loading weights"):
black_net = dual_net.DualNetwork(black_model)
white_net = dual_net.DualNetwork(white_model)
with timer("%d games" % games):
players = evaluation.play_match(black_net, white_net, games, readouts,
verbose)
for idx, p in enumerate(players):
fname = "{:s}-vs-{:s}-{:d}".format(black_net.name, white_net.name, idx)
with open(os.path.join(output_dir, fname + '.sgf'), 'w') as f:
f.write(
sgf_wrapper.make_sgf(p[0].position.recent,
p[0].make_result_string(p[0].position),
black_name=os.path.basename(black_model),
white_name=os.path.basename(white_model)))
def evaluate(
black_model: 'The path to the model to play black',
white_model: 'The path to the model to play white',
output_dir: 'Where to write the evaluation results'='data/evaluate/sgf',
readouts: 'How many readouts to make per move.'=400,
games: 'the number of games to play'=16,
verbose: 'How verbose the players should be (see selfplay)' = 1):
black_model = os.path.abspath(black_model)
white_model = os.path.abspath(white_model)
with timer("Loading weights"):
black_net = dual_net.DualNetwork(black_model)
white_net = dual_net.DualNetwork(white_model)
with timer("%d games" % games):
players = evaluation.play_match(
black_net, white_net, games, readouts, verbose)
for idx, p in enumerate(players):
fname = "{:s}-vs-{:s}-{:d}".format(black_net.name, white_net.name, idx)
with open(os.path.join(output_dir, fname + '.sgf'), 'w') as f:
f.write(sgf_wrapper.make_sgf(p[0].position.recent,
p[0].make_result_string(
p[0].position),
black_name=os.path.basename(
black_model),
white_name=os.path.basename(white_model)))
def to_sgf(self):
assert self.result_string is not None
pos = self.root.position
if self.comments:
self.comments[0] = ("Resign Threshold: %0.3f\n" %
self.resign_threshold) + self.comments[0]
return sgf_wrapper.make_sgf(pos.recent, self.result_string,
white_name=self.network.name or "Unknown",
black_name=self.network.name or "Unknown",
comments=self.comments)
def to_sgf(self):
assert self.result_string is not None
pos = self.root.position
if self.comments:
self.comments[0] = ("Resign Threshold: %0.3f\n" %
self.resign_threshold) + self.comments[0]
return sgf_wrapper.make_sgf(pos.recent, self.result_string,
white_name=self.network.name or "Unknown",
black_name=self.network.name or "Unknown",
comments=self.comments)
def test_make_sgf(self):
all_positions = list(replay_sgf(NO_HANDICAP_SGF))
last_position, _, metadata = all_positions[-1]
back_to_sgf = make_sgf(
last_position.recent,
last_position.score(),
boardsize=metadata.board_size,
komi=last_position.komi,
)
reconstructed_positions = list(replay_sgf(back_to_sgf))
last_position2, _, _ = reconstructed_positions[-1]
self.assertEqualPositions(last_position, last_position2)
def to_sgf(self, use_comments=True):
assert self.result_string is not None
pos = self.root.position
if use_comments:
comments = self.comments or ['No comments.']
comments[0] = ("Resign Threshold: %0.3f\n" %
self.resign_threshold) + comments[0]
else:
comments = []
return sgf_wrapper.make_sgf(pos.recent, self.result_string,
white_name=self.network.name or "Unknown",
black_name=self.network.name or "Unknown",
comments=comments)
def test_make_sgf(self):
all_positions = list(replay_sgf(NO_HANDICAP_SGF))
last_position, _, metadata = all_positions[-1]
back_to_sgf = make_sgf(
last_position.recent,
last_position.score(),
boardsize=metadata.board_size,
komi=last_position.komi,
)
reconstructed_positions = list(replay_sgf(back_to_sgf))
last_position2, _, _ = reconstructed_positions[-1]
self.assertEqualPositions(last_position, last_position2)
def to_sgf(self, use_comments=True):
assert self.result_string is not None
pos = self.root.position
if use_comments:
comments = self.comments or ['No comments.']
comments[0] = ('Resign Threshold: %0.3f\n' %
self.resign_threshold) + comments[0]
else:
comments = []
return sgf_wrapper.make_sgf(
self.board_size, pos.recent, self.result_string,
white_name=os.path.basename(self.network.save_file) or 'Unknown',
black_name=os.path.basename(self.network.save_file) or 'Unknown',
comments=comments)
def to_sgf(self, use_comments=True):
assert self.result_string is not None
pos = self.root.position
if use_comments:
comments = self.comments or ['No comments.']
comments[0] = ('Resign Threshold: %0.3f\n' %
self.resign_threshold) + comments[0]
else:
comments = []
return sgf_wrapper.make_sgf(
self.board_size, pos.recent, self.result_string,
white_name=os.path.basename(self.network.save_file) or 'Unknown',
black_name=os.path.basename(self.network.save_file) or 'Unknown',
comments=comments)
def test_make_sgf(self):
all_pwcs = list(replay_sgf(NO_HANDICAP_SGF))
second_last_position, last_move, _ = all_pwcs[-1]
last_position = second_last_position.play_move(last_move)
back_to_sgf = make_sgf(
last_position.recent,
last_position.score(),
komi=last_position.komi,
)
reconstructed_positions = list(replay_sgf(back_to_sgf))
second_last_position2, last_move2, _ = reconstructed_positions[-1]
last_position2 = second_last_position2.play_move(last_move2)
self.assertEqualPositions(last_position, last_position2)
def test_make_sgf(self):
all_pwcs = list(replay_sgf(NO_HANDICAP_SGF))
second_last_position, last_move, _ = all_pwcs[-1]
last_position = second_last_position.play_move(last_move)
back_to_sgf = make_sgf(
last_position.recent,
last_position.score(),
komi=last_position.komi,
)
reconstructed_positions = list(replay_sgf(back_to_sgf))
second_last_position2, last_move2, _ = reconstructed_positions[-1]
last_position2 = second_last_position2.play_move(last_move2)
self.assertEqualPositions(last_position, last_position2)
def play_match(black_net, white_net, games, readouts, sgf_dir, verbosity):
"""Plays matches between two neural nets.
black_net: Instance of minigo.DualNetwork, a wrapper around a tensorflow
convolutional network.
white_net: Instance of the minigo.DualNetwork.
games: number of games to play. We play all the games at the same time.
sgf_dir: directory to write the sgf results.
readouts: number of readouts to perform for each step in each game.
"""
# For n games, we create lists of n black and n white players
black = MCTSPlayer(
black_net, verbosity=verbosity, two_player_mode=True, num_parallel=SIMULTANEOUS_LEAVES)
white = MCTSPlayer(
white_net, verbosity=verbosity, two_player_mode=True, num_parallel=SIMULTANEOUS_LEAVES)
black_name = os.path.basename(black_net.save_file)
white_name = os.path.basename(white_net.save_file)
winners = []
for i in range(games):
num_move = 0 # The move number of the current game
black.initialize_game()
white.initialize_game()
while True:
start = time.time()
active = white if num_move % 2 else black
inactive = black if num_move % 2 else white
current_readouts = active.root.N
while active.root.N < current_readouts + readouts:
active.tree_search()
# print some stats on the search
if verbosity >= 3:
print(active.root.position)
# First, check the roots for hopeless games.
if active.should_resign(): # Force resign
active.set_result(-1 *
active.root.position.to_play, was_resign=True)
inactive.set_result(
active.root.position.to_play, was_resign=True)
if active.is_done():
fname = "{:d}-{:s}-vs-{:s}-{:d}.sgf".format(int(time.time()),
white_name, black_name, i)
if active.result_string is None:
# This is an exceptionally rare corner case where we don't get a winner.
# Our temporary solution is to just drop this game.
break
winners.append(active.result_string[0])
with open(os.path.join(sgf_dir, fname), 'w') as _file:
sgfstr = sgf_wrapper.make_sgf(active.position.recent,
active.result_string, black_name=black_name,
white_name=white_name)
_file.write(sgfstr)
print("Finished game", i, active.result_string)
break
move = active.pick_move()
# print('DBUG Picked move: ', move, active, num_move)
active.play_move(move)
inactive.play_move(move)
dur = time.time() - start
num_move += 1
if (verbosity > 1) or (verbosity == 1 and num_move % 10 == 9):
timeper = (dur / readouts) * 100.0
print(active.root.position)
print("%d: %d readouts, %.3f s/100. (%.2f sec)" % (num_move,
readouts,
timeper,
dur))
return winners
def play_match(black_net, white_net, games, sgf_dir, verbosity):
"""Plays matches between two neural nets.
black_net: Instance of minigo.DualNetwork, a wrapper around a tensorflow
convolutional network.
white_net: Instance of the minigo.DualNetwork.
games: number of games to play. We play all the games at the same time.
sgf_dir: directory to write the sgf results.
"""
readouts = flags.FLAGS.num_readouts # Flag defined in strategies.py
black = MCTSPlayer(
black_net, verbosity=verbosity, two_player_mode=True)
white = MCTSPlayer(
white_net, verbosity=verbosity, two_player_mode=True)
black_name = os.path.basename(black_net.save_file)
white_name = os.path.basename(white_net.save_file)
for i in range(games):
num_move = 0 # The move number of the current game
black.initialize_game()
white.initialize_game()
while True:
start = time.time()
active = white if num_move % 2 else black
inactive = black if num_move % 2 else white
current_readouts = active.root.N
while active.root.N < current_readouts + readouts:
active.tree_search()
# print some stats on the search
if verbosity >= 3:
print(active.root.position)
# First, check the roots for hopeless games.
if active.should_resign(): # Force resign
active.set_result(-1 *
active.root.position.to_play, was_resign=True)
inactive.set_result(
active.root.position.to_play, was_resign=True)
if active.is_done():
fname = "{:d}-{:s}-vs-{:s}-{:d}.sgf".format(int(time.time()),
white_name, black_name, i)
with gfile.GFile(os.path.join(sgf_dir, fname), 'w') as _file:
sgfstr = sgf_wrapper.make_sgf(active.position.recent,
active.result_string, black_name=black_name,
white_name=white_name)
_file.write(sgfstr)
print("Finished game", i, active.result_string)
break
move = active.pick_move()
active.play_move(move)
inactive.play_move(move)
dur = time.time() - start
num_move += 1
if (verbosity > 1) or (verbosity == 1 and num_move % 10 == 9):
timeper = (dur / readouts) * 100.0
print(active.root.position)
print("%d: %d readouts, %.3f s/100. (%.2f sec)" % (num_move,
readouts,
timeper,
dur))
def play_match(black_model, white_model, games, sgf_dir):
"""Plays matches between two neural nets.
Args:
black_model: Path to the model for black player
white_model: Path to the model for white player
"""
with utils.logged_timer("Loading weights"):
black_net = dual_net.DualNetwork(black_model)
white_net = dual_net.DualNetwork(white_model)
readouts = FLAGS.num_readouts
black = MCTSPlayer(black_net, two_player_mode=True)
white = MCTSPlayer(white_net, two_player_mode=True)
black_name = os.path.basename(black_net.save_file)
white_name = os.path.basename(white_net.save_file)
for i in range(games):
num_move = 0 # The move number of the current game
for player in [black, white]:
player.initialize_game()
first_node = player.root.select_leaf()
prob, val = player.network.run(first_node.position)
first_node.incorporate_results(prob, val, first_node)
while True:
start = time.time()
active = white if num_move % 2 else black
inactive = black if num_move % 2 else white
current_readouts = active.root.N
while active.root.N < current_readouts + readouts:
active.tree_search()
# print some stats on the search
if FLAGS.verbose >= 3:
print(active.root.position)
# First, check the roots for hopeless games.
if active.should_resign(): # Force resign
active.set_result(-1 *
active.root.position.to_play, was_resign=True)
inactive.set_result(
active.root.position.to_play, was_resign=True)
if active.is_done():
fname = "{:d}-{:s}-vs-{:s}-{:d}.sgf".format(int(time.time()),
white_name, black_name, i)
with gfile.GFile(os.path.join(sgf_dir, fname), 'w') as _file:
sgfstr = sgf_wrapper.make_sgf(active.position.recent,
active.result_string, black_name=black_name,
white_name=white_name)
_file.write(sgfstr)
print("Finished game", i, active.result_string)
break
move = active.pick_move()
active.play_move(move)
inactive.play_move(move)
dur = time.time() - start
num_move += 1
if (FLAGS.verbose > 1) or (FLAGS.verbose == 1 and num_move % 10 == 9):
timeper = (dur / readouts) * 100.0
print(active.root.position)
print("%d: %d readouts, %.3f s/100. (%.2f sec)" % (num_move,
readouts,
timeper,
dur))
def play_match(black_net, white_net, games, readouts, sgf_dir, verbosity):
"""Plays matches between two neural nets.
black_net: Instance of minigo.DualNetwork, a wrapper around a tensorflow
convolutional network.
white_net: Instance of the minigo.DualNetwork.
games: number of games to play. We play all the games at the same time.
sgf_dir: directory to write the sgf results.
readouts: number of readouts to perform for each step in each game.
"""
# For n games, we create lists of n black and n white players
black = MCTSPlayer(black_net,
verbosity=verbosity,
two_player_mode=True,
num_parallel=SIMULTANEOUS_LEAVES)
white = MCTSPlayer(white_net,
verbosity=verbosity,
two_player_mode=True,
num_parallel=SIMULTANEOUS_LEAVES)
black_name = os.path.basename(black_net.save_file)
white_name = os.path.basename(white_net.save_file)
winners = []
for i in range(games):
num_move = 0 # The move number of the current game
black.initialize_game()
white.initialize_game()
while True:
start = time.time()
active = white if num_move % 2 else black
inactive = black if num_move % 2 else white
current_readouts = active.root.N
while active.root.N < current_readouts + readouts:
active.tree_search()
# print some stats on the search
if verbosity >= 3:
print(active.root.position)
# First, check the roots for hopeless games.
if active.should_resign(): # Force resign
active.set_result(-1 * active.root.position.to_play,
was_resign=True)
inactive.set_result(active.root.position.to_play,
was_resign=True)
if active.is_done():
fname = "{:d}-{:s}-vs-{:s}-{:d}.sgf".format(
int(time.time()), white_name, black_name, i)
if active.result_string is None:
# This is an exceptionally rare corner case where we don't get a winner.
# Our temporary solution is to just drop this game.
break
winners.append(active.result_string[0])
with open(os.path.join(sgf_dir, fname), 'w') as _file:
sgfstr = sgf_wrapper.make_sgf(active.position.recent,
active.result_string,
black_name=black_name,
white_name=white_name)
_file.write(sgfstr)
print("Finished game", i, active.result_string)
break
move = active.pick_move()
# print('DBUG Picked move: ', move, active, num_move)
active.play_move(move)
inactive.play_move(move)
dur = time.time() - start
num_move += 1
if (verbosity > 1) or (verbosity == 1 and num_move % 10 == 9):
timeper = (dur / readouts) * 100.0
print(active.root.position)
print("%d: %d readouts, %.3f s/100. (%.2f sec)" %
(num_move, readouts, timeper, dur))
return winners
def play_match(params, black_net, white_net, games, readouts, sgf_dir,
verbosity):
"""Plays matches between two neural nets.
One net that wins by a margin of 55% will be the winner.
Args:
params: An object of hyperparameters.
black_net: Instance of the DualNetRunner class to play as black.
white_net: Instance of the DualNetRunner class to play as white.
games: Number of games to play. We play all the games at the same time.
readouts: Number of readouts to perform for each step in each game.
sgf_dir: Directory to write the sgf results.
verbosity: Verbosity to show evaluation process.
Returns:
'B' is the winner is black_net, otherwise 'W'.
"""
# For n games, we create lists of n black and n white players
black = MCTSPlayer(params.board_size,
black_net,
verbosity=verbosity,
two_player_mode=True,
num_parallel=params.simultaneous_leaves)
white = MCTSPlayer(params.board_size,
white_net,
verbosity=verbosity,
two_player_mode=True,
num_parallel=params.simultaneous_leaves)
black_name = os.path.basename(black_net.save_file)
white_name = os.path.basename(white_net.save_file)
black_win_counts = 0
white_win_counts = 0
for i in range(games):
num_move = 0 # The move number of the current game
black.initialize_game()
white.initialize_game()
while True:
start = time.time()
active = white if num_move % 2 else black
inactive = black if num_move % 2 else white
current_readouts = active.root.N
while active.root.N < current_readouts + readouts:
active.tree_search()
# print some stats on the search
if verbosity >= 3:
print(active.root.position)
# First, check the roots for hopeless games.
if active.should_resign(): # Force resign
active.set_result(-active.root.position.to_play,
was_resign=True)
inactive.set_result(active.root.position.to_play,
was_resign=True)
if active.is_done():
fname = '{:d}-{:s}-vs-{:s}-{:d}.sgf'.format(
int(time.time()), white_name, black_name, i)
with open(os.path.join(sgf_dir, fname), 'w') as f:
sgfstr = sgf_wrapper.make_sgf(params.board_size,
active.position.recent,
active.result_string,
black_name=black_name,
white_name=white_name)
f.write(sgfstr)
print('Finished game', i, active.result_string)
if active.result_string is not None:
if active.result_string[0] == 'B':
black_win_counts += 1
elif active.result_string[0] == 'W':
white_win_counts += 1
break
move = active.pick_move()
active.play_move(move)
inactive.play_move(move)
dur = time.time() - start
num_move += 1
if (verbosity > 1) or (verbosity == 1 and num_move % 10 == 9):
timeper = (dur / readouts) * 100.0
print(active.root.position)
print('{:d}: {:d} readouts, {:.3f} s/100. ({:.2f} sec)'.format(
num_move, readouts, timeper, dur))
if (black_win_counts - white_win_counts) > params.eval_win_rate * games:
return go.BLACK_NAME
else:
return go.WHITE_NAME
