def validate_hourly(working_dir, validate_name=None):
""" compiles a list of games based on the new hourly directory format. Then
calls validate on it """
holdout_dirs = gfile.ListDirectory(fsdb.holdout_dir())
holdout_files = (os.path.join(fsdb.holdout_dir(), d, f)
for d in reversed(gfile.ListDirectory(fsdb.holdout_dir()))
for f in gfile.ListDirectory(os.path.join(fsdb.holdout_dir(),d))
if gfile.IsDirectory(os.path.join(fsdb.holdout_dir(),d)))
holdout_files = list(itertools.islice(holdout_files, 20000))
random.shuffle(holdout_files)
dual_net.validate(holdout_files)
def validate(working_dir, model_num=None, validate_name=None):
""" Runs validate on the directories up to the most recent model, or up to
(but not including) the model specified by `model_num`
"""
if model_num is None:
model_num, model_name = fsdb.get_latest_model()
else:
model_num = int(model_num)
model_name = fsdb.get_model(model_num)
# Model N was trained on games up through model N-2, so the validation set
# should only be for models through N-2 as well, thus the (model_num - 1)
# term.
models = list(
filter(lambda num_name: num_name[0] < (model_num - 1),
fsdb.get_models()))
# Run on the most recent 50 generations,
# TODO(brianklee): make this hyperparameter dependency explicit/not hardcoded
holdout_dirs = [
os.path.join(fsdb.holdout_dir(), pair[1]) for pair in models[-50:]
]
main.validate(working_dir,
*holdout_dirs,
checkpoint_name=os.path.join(fsdb.models_dir(), model_name),
validate_name=validate_name)
def selfplay(verbose=2):
_, model_name = fsdb.get_latest_model()
games = gfile.Glob(os.path.join(fsdb.selfplay_dir(), model_name, '*.zz'))
if len(games) > MAX_GAMES_PER_GENERATION:
print("{} has enough games ({})".format(model_name, len(games)))
time.sleep(10 * 60)
sys.exit(1)
print("Playing a game with model {}".format(model_name))
model_save_path = os.path.join(fsdb.models_dir(), model_name)
game_output_dir = os.path.join(fsdb.selfplay_dir(), model_name)
game_holdout_dir = os.path.join(fsdb.holdout_dir(), model_name)
sgf_dir = os.path.join(fsdb.sgf_dir(), model_name)
main.selfplay(
load_file=model_save_path,
output_dir=game_output_dir,
holdout_dir=game_holdout_dir,
output_sgf=sgf_dir,
holdout_pct=HOLDOUT_PCT,
verbose=verbose,
)
