def validate_hourly(working_dir, validate_name=None): """ compiles a list of games based on the new hourly directory format. Then calls validate on it """ holdout_dirs = gfile.ListDirectory(fsdb.holdout_dir()) holdout_files = (os.path.join(fsdb.holdout_dir(), d, f) for d in reversed(gfile.ListDirectory(fsdb.holdout_dir())) for f in gfile.ListDirectory(os.path.join(fsdb.holdout_dir(),d)) if gfile.IsDirectory(os.path.join(fsdb.holdout_dir(),d))) holdout_files = list(itertools.islice(holdout_files, 20000)) random.shuffle(holdout_files) dual_net.validate(holdout_files)
def validate(working_dir, model_num=None, validate_name=None): """ Runs validate on the directories up to the most recent model, or up to (but not including) the model specified by `model_num` """ if model_num is None: model_num, model_name = fsdb.get_latest_model() else: model_num = int(model_num) model_name = fsdb.get_model(model_num) # Model N was trained on games up through model N-2, so the validation set # should only be for models through N-2 as well, thus the (model_num - 1) # term. models = list( filter(lambda num_name: num_name[0] < (model_num - 1), fsdb.get_models())) # Run on the most recent 50 generations, # TODO(brianklee): make this hyperparameter dependency explicit/not hardcoded holdout_dirs = [ os.path.join(fsdb.holdout_dir(), pair[1]) for pair in models[-50:] ] main.validate(working_dir, *holdout_dirs, checkpoint_name=os.path.join(fsdb.models_dir(), model_name), validate_name=validate_name)
def selfplay(verbose=2): _, model_name = fsdb.get_latest_model() games = gfile.Glob(os.path.join(fsdb.selfplay_dir(), model_name, '*.zz')) if len(games) > MAX_GAMES_PER_GENERATION: print("{} has enough games ({})".format(model_name, len(games))) time.sleep(10 * 60) sys.exit(1) print("Playing a game with model {}".format(model_name)) model_save_path = os.path.join(fsdb.models_dir(), model_name) game_output_dir = os.path.join(fsdb.selfplay_dir(), model_name) game_holdout_dir = os.path.join(fsdb.holdout_dir(), model_name) sgf_dir = os.path.join(fsdb.sgf_dir(), model_name) main.selfplay( load_file=model_save_path, output_dir=game_output_dir, holdout_dir=game_holdout_dir, output_sgf=sgf_dir, holdout_pct=HOLDOUT_PCT, verbose=verbose, )