def selfplay(
load_file: "The path to the network model files",
output_dir: "Where to write the games" = "data/selfplay",
output_sgf: "Where to write the sgfs" = "sgf/",
readouts: 'How many simulations to run per move' = 100,
verbose: '>=2 will print debug info, >=3 will print boards' = 1,
resign_threshold: 'absolute value of threshold to resign at' = 0.95):
_ensure_dir_exists(output_sgf)
_ensure_dir_exists(output_dir)
with timer("Loading weights from %s ... " % load_file):
network = dual_net.DualNetwork(load_file)
network.name = os.path.basename(load_file)
with timer("Playing game"):
player = selfplay_mcts.play(network, readouts, resign_threshold,
verbose)
output_name = '{}-{}'.format(int(time.time()), socket.gethostname())
game_data = player.extract_data()
with gfile.GFile(os.path.join(output_sgf, '{}.sgf'.format(output_name)),
'w') as f:
f.write(player.to_sgf())
fname = os.path.join(output_dir, "{}.tfrecord.zz".format(output_name))
preprocessing.make_dataset_from_selfplay(game_data, fname)
def selfplay(
load_file: "The path to the network model files",
output_dir: "Where to write the games"="data/selfplay",
holdout_dir: "Where to write the games"="data/holdout",
output_sgf: "Where to write the sgfs"="sgf/",
readouts: 'How many simulations to run per move'=100,
verbose: '>=2 will print debug info, >=3 will print boards' = 1,
resign_threshold: 'absolute value of threshold to resign at' = 0.95,
holdout_pct: 'how many games to hold out for evaluation' = 0.05):
_ensure_dir_exists(output_sgf)
_ensure_dir_exists(output_dir)
with timer("Loading weights from %s ... " % load_file):
network = dual_net.DualNetwork(load_file)
network.name = os.path.basename(load_file)
with timer("Playing game"):
player = selfplay_mcts.play(
network, readouts, resign_threshold, verbose)
output_name = '{}-{}'.format(int(time.time()), socket.gethostname())
game_data = player.extract_data()
with gfile.GFile(os.path.join(output_sgf, '{}.sgf'.format(output_name)), 'w') as f:
f.write(player.to_sgf())
tf_examples = preprocessing.make_dataset_from_selfplay(game_data)
# Hold out 5% of games for evaluation.
if random.random() < holdout_pct:
fname = os.path.join(holdout_dir, "{}.tfrecord.zz".format(output_name))
else:
fname = os.path.join(output_dir, "{}.tfrecord.zz".format(output_name))
preprocessing.write_tf_examples(fname, tf_examples)
def run_game(load_file,
selfplay_dir=None,
holdout_dir=None,
sgf_dir=None,
holdout_pct=0.05):
'''Takes a played game and record results and game data.'''
if sgf_dir is not None:
minimal_sgf_dir = os.path.join(sgf_dir, 'clean')
full_sgf_dir = os.path.join(sgf_dir, 'full')
utils.ensure_dir_exists(minimal_sgf_dir)
utils.ensure_dir_exists(full_sgf_dir)
if selfplay_dir is not None:
utils.ensure_dir_exists(selfplay_dir)
utils.ensure_dir_exists(holdout_dir)
with utils.logged_timer("Loading weights from %s ... " % load_file):
network = dual_net.DualNetwork(load_file)
with utils.logged_timer("Playing game"):
player = play(network)
output_name = '{}-{}'.format(int(time.time()), socket.gethostname())
game_data = player.extract_data()
if sgf_dir is not None:
with gfile.GFile(
os.path.join(minimal_sgf_dir, '{}.sgf'.format(output_name)),
'w') as f:
f.write(player.to_sgf(use_comments=False))
with gfile.GFile(
os.path.join(full_sgf_dir, '{}.sgf'.format(output_name)),
'w') as f:
f.write(player.to_sgf())
tf_examples = preprocessing.make_dataset_from_selfplay(game_data)
if selfplay_dir is not None:
# Hold out 5% of games for validation.
if random.random() < holdout_pct:
fname = os.path.join(holdout_dir,
"{}.tfrecord.zz".format(output_name))
else:
fname = os.path.join(selfplay_dir,
"{}.tfrecord.zz".format(output_name))
preprocessing.write_tf_examples(fname, tf_examples)
def selfplay(
load_file: "The path to the network model files",
output_dir: "Where to write the games" = "data/selfplay",
holdout_dir: "Where to write the games" = "data/holdout",
output_sgf: "Where to write the sgfs" = "sgf/",
readouts: 'How many simulations to run per move' = 100,
verbose: '>=2 will print debug info, >=3 will print boards' = 1,
resign_threshold: 'absolute value of threshold to resign at' = 0.95,
holdout_pct: 'how many games to hold out for validation' = 0.05):
qmeas.start_time('selfplay')
clean_sgf = os.path.join(output_sgf, 'clean')
full_sgf = os.path.join(output_sgf, 'full')
_ensure_dir_exists(clean_sgf)
_ensure_dir_exists(full_sgf)
_ensure_dir_exists(output_dir)
_ensure_dir_exists(holdout_dir)
with timer("Loading weights from %s ... " % load_file):
network = dual_net.DualNetwork(load_file)
with timer("Playing game"):
player = selfplay_mcts.play(network, readouts, resign_threshold,
verbose)
output_name = '{}-{}'.format(int(time.time() * 1000 * 1000),
socket.gethostname())
game_data = player.extract_data()
with gfile.GFile(os.path.join(clean_sgf, '{}.sgf'.format(output_name)),
'w') as f:
f.write(player.to_sgf(use_comments=False))
with gfile.GFile(os.path.join(full_sgf, '{}.sgf'.format(output_name)),
'w') as f:
f.write(player.to_sgf())
tf_examples = preprocessing.make_dataset_from_selfplay(game_data)
# Hold out 5% of games for evaluation.
if random.random() < holdout_pct:
fname = os.path.join(holdout_dir, "{}.tfrecord.zz".format(output_name))
else:
fname = os.path.join(output_dir, "{}.tfrecord.zz".format(output_name))
preprocessing.write_tf_examples(fname, tf_examples)
qmeas.stop_time('selfplay')
def selfplay(selfplay_dirs, selfplay_model, params):
"""Perform selfplay with a specific model.
Args:
selfplay_dirs: A dict to specify the directories used in selfplay.
selfplay_dirs = {
'output_dir': output_dir,
'holdout_dir': holdout_dir,
'clean_sgf': clean_sgf,
'full_sgf': full_sgf
}
selfplay_model: The actual Dualnet runner for selfplay.
params: A MiniGoParams instance of hyperparameters for the model.
"""
with utils.logged_timer('Playing game'):
player = selfplay_mcts.play(params.board_size, selfplay_model,
params.selfplay_readouts,
params.selfplay_resign_threshold,
params.simultaneous_leaves,
params.selfplay_verbose)
output_name = '{}-{}'.format(int(time.time()), socket.gethostname())
def _write_sgf_data(dir_sgf, use_comments):
with tf.gfile.GFile(
os.path.join(dir_sgf, '{}.sgf'.format(output_name)), 'w') as f:
f.write(player.to_sgf(use_comments=use_comments))
_write_sgf_data(selfplay_dirs['clean_sgf'], use_comments=False)
_write_sgf_data(selfplay_dirs['full_sgf'], use_comments=True)
game_data = player.extract_data()
tf_examples = preprocessing.make_dataset_from_selfplay(game_data, params)
# Hold out 5% of games for evaluation.
if random.random() < params.holdout_pct:
fname = os.path.join(selfplay_dirs['holdout_dir'],
output_name + _TF_RECORD_SUFFIX)
else:
fname = os.path.join(selfplay_dirs['output_dir'],
output_name + _TF_RECORD_SUFFIX)
preprocessing.write_tf_examples(fname, tf_examples)
def selfplay(selfplay_dirs, selfplay_model, params):
"""Perform selfplay with a specific model.
Args:
selfplay_dirs: A dict to specify the directories used in selfplay.
selfplay_dirs = {
'output_dir': output_dir,
'holdout_dir': holdout_dir,
'clean_sgf': clean_sgf,
'full_sgf': full_sgf
}
selfplay_model: The actual Dualnet runner for selfplay.
params: A MiniGoParams instance of hyperparameters for the model.
"""
with utils.logged_timer('Playing game'):
player = selfplay_mcts.play(
params.board_size, selfplay_model, params.selfplay_readouts,
params.selfplay_resign_threshold, params.simultaneous_leaves,
params.selfplay_verbose)
output_name = '{}-{}'.format(int(time.time()), socket.gethostname())
def _write_sgf_data(dir_sgf, use_comments):
with tf.gfile.GFile(
os.path.join(dir_sgf, '{}.sgf'.format(output_name)), 'w') as f:
f.write(player.to_sgf(use_comments=use_comments))
_write_sgf_data(selfplay_dirs['clean_sgf'], use_comments=False)
_write_sgf_data(selfplay_dirs['full_sgf'], use_comments=True)
game_data = player.extract_data()
tf_examples = preprocessing.make_dataset_from_selfplay(game_data, params)
# Hold out 5% of games for evaluation.
if random.random() < params.holdout_pct:
fname = os.path.join(
selfplay_dirs['holdout_dir'], output_name + _TF_RECORD_SUFFIX)
else:
fname = os.path.join(
selfplay_dirs['output_dir'], output_name + _TF_RECORD_SUFFIX)
preprocessing.write_tf_examples(fname, tf_examples)
def selfplay_cache_model(
network: "The path to the network model files",
output_dir: "Where to write the games"="data/selfplay",
holdout_dir: "Where to write the games"="data/holdout",
output_sgf: "Where to write the sgfs"="sgf/",
readouts: 'How many simulations to run per move'=100,
verbose: '>=2 will print debug info, >=3 will print boards' = 1,
resign_threshold: 'absolute value of threshold to resign at' = 0.95,
holdout_pct: 'how many games to hold out for validation' = 0.05):
qmeas.start_time('selfplay')
clean_sgf = os.path.join(output_sgf, 'clean')
full_sgf = os.path.join(output_sgf, 'full')
_ensure_dir_exists(clean_sgf)
_ensure_dir_exists(full_sgf)
_ensure_dir_exists(output_dir)
_ensure_dir_exists(holdout_dir)
with timer("Playing game"):
player = selfplay_mcts.play(
network, readouts, resign_threshold, verbose)
output_name = '{}-{}'.format(int(time.time() * 1000 * 1000), socket.gethostname())
game_data = player.extract_data()
with gfile.GFile(os.path.join(clean_sgf, '{}.sgf'.format(output_name)), 'w') as f:
f.write(player.to_sgf(use_comments=False))
with gfile.GFile(os.path.join(full_sgf, '{}.sgf'.format(output_name)), 'w') as f:
f.write(player.to_sgf())
tf_examples = preprocessing.make_dataset_from_selfplay(game_data)
# Hold out 5% of games for evaluation.
if random.random() < holdout_pct:
fname = os.path.join(holdout_dir, "{}.tfrecord.zz".format(output_name))
else:
fname = os.path.join(output_dir, "{}.tfrecord.zz".format(output_name))
preprocessing.write_tf_examples(fname, tf_examples)
qmeas.stop_time('selfplay')
def selfplay(model_name, trained_models_dir, selfplay_dir, holdout_dir,
sgf_dir, params):
"""Perform selfplay with a specific model.
Args:
model_name: The name of the model used for selfplay.
trained_models_dir: The path to the model files.
selfplay_dir: Where to write the games. Set as 'base_dir/data/selfplay/'.
holdout_dir: Where to write the holdout data. Set as
'base_dir/data/holdout/'.
sgf_dir: Where to write the sgf (Smart Game Format) files. Set as
'base_dir/sgf/'.
params: An object of hyperparameters for the model.
"""
print('Playing a game with model {}'.format(model_name))
# Set paths for the model with 'model_name'
model_path = os.path.join(trained_models_dir, model_name)
output_dir = os.path.join(selfplay_dir, model_name)
holdout_dir = os.path.join(holdout_dir, model_name)
# clean_sgf is to write sgf file without comments.
# full_sgf is to write sgf file with comments.
clean_sgf = os.path.join(sgf_dir, model_name, 'clean')
full_sgf = os.path.join(sgf_dir, model_name, 'full')
_ensure_dir_exists(output_dir)
_ensure_dir_exists(holdout_dir)
_ensure_dir_exists(clean_sgf)
_ensure_dir_exists(full_sgf)
with utils.logged_timer('Loading weights from {} ... '.format(model_path)):
network = dualnet.DualNetRunner(model_path, params)
with utils.logged_timer('Playing game'):
player = selfplay_mcts.play(params.board_size, network,
params.selfplay_readouts,
params.selfplay_resign_threshold,
params.simultaneous_leaves,
params.selfplay_verbose)
output_name = '{}-{}'.format(int(time.time()), socket.gethostname())
def _write_sgf_data(dir_sgf, use_comments):
with tf.gfile.GFile(
os.path.join(dir_sgf, '{}.sgf'.format(output_name)), 'w') as f:
f.write(player.to_sgf(use_comments=use_comments))
_write_sgf_data(clean_sgf, use_comments=False)
_write_sgf_data(full_sgf, use_comments=True)
game_data = player.extract_data()
tf_examples = preprocessing.make_dataset_from_selfplay(game_data, params)
# Hold out 5% of games for evaluation.
if random.random() < params.holdout_pct:
fname = os.path.join(holdout_dir,
('{}' + _TF_RECORD_SUFFIX).format(output_name))
else:
fname = os.path.join(output_dir,
('{}' + _TF_RECORD_SUFFIX).format(output_name))
preprocessing.write_tf_examples(fname, tf_examples)
_ensure_dir_exists(output_dir)
with timer("Loading weights from %s ... " % load_file):
network = dual_net.DualNetwork(load_file)
network.name = os.path.basename(load_file)
with timer("Playing game"):
player = selfplay_mcts.play(
network, readouts, resign_threshold, verbose)
output_name = '{}-{}'.format(int(time.time()), socket.gethostname())
game_data = player.extract_data()
with gfile.GFile(os.path.join(output_sgf, '{}.sgf'.format(output_name)), 'w') as f:
f.write(player.to_sgf())
tf_examples = preprocessing.make_dataset_from_selfplay(game_data)
# Hold out 5% of games for evaluation.
if random.random() < holdout_pct:
fname = os.path.join(holdout_dir, "{}.tfrecord.zz".format(output_name))
else:
fname = os.path.join(output_dir, "{}.tfrecord.zz".format(output_name))
preprocessing.write_tf_examples(fname, tf_examples)
def gather(
input_directory: 'where to look for games'='data/selfplay/',
output_directory: 'where to put collected games'='data/training_chunks/',
examples_per_record: 'how many tf.examples to gather in each chunk'=20000):
_ensure_dir_exists(output_directory)
