def launch_eval(black_num=0, white_num=0):
if black_num <= 0 or white_num <= 0:
print("Need real model numbers")
return
b = fsdb.get_model(black_num)
w = fsdb.get_model(white_num)
b_model_path = os.path.join(fsdb.models_dir(), b)
w_model_path = os.path.join(fsdb.models_dir(), w)
kubernetes.config.load_kube_config()
configuration = kubernetes.client.Configuration()
api_instance = kubernetes.client.BatchV1Api(
kubernetes.client.ApiClient(configuration))
raw_job_conf = open("cluster/evaluator/gpu-evaluator.yaml").read()
env_job_conf = os.path.expandvars(raw_job_conf)
t = jinja2.Template(env_job_conf)
job_conf = yaml.load(t.render({'white': w_model_path,
'black': b_model_path,
'wnum': white_num,
'bnum': black_num}))
resp = api_instance.create_namespaced_job('default', body=job_conf)
job_conf = yaml.load(t.render({'white': b_model_path,
'black': w_model_path,
'wnum': black_num,
'bnum': white_num}))
resp = api_instance.create_namespaced_job('default', body=job_conf)
def validate(working_dir, model_num=None, validate_name=None):
""" Runs validate on the directories up to the most recent model, or up to
(but not including) the model specified by `model_num`
"""
if model_num is None:
model_num, model_name = fsdb.get_latest_model()
else:
model_num = int(model_num)
model_name = fsdb.get_model(model_num)
# Model N was trained on games up through model N-2, so the validation set
# should only be for models through N-2 as well, thus the (model_num - 1)
# term.
models = list(
filter(lambda num_name: num_name[0] < (model_num - 1),
fsdb.get_models()))
# Run on the most recent 50 generations,
# TODO(brianklee): make this hyperparameter dependency explicit/not hardcoded
holdout_dirs = [
os.path.join(fsdb.holdout_dir(), pair[1]) for pair in models[-50:]
]
main.validate(working_dir,
*holdout_dirs,
checkpoint_name=os.path.join(fsdb.models_dir(), model_name),
validate_name=validate_name)
def same_run_eval(black_num=0, white_num=0):
"""Shorthand to spawn a job matching up two models from the same run,
identified by their model number """
if black_num <= 0 or white_num <= 0:
print("Need real model numbers")
return
b = fsdb.get_model(black_num)
w = fsdb.get_model(white_num)
bucket = fsdb.eval_dir
b_model_path = os.path.join(fsdb.models_dir(), b)
w_model_path = os.path.join(fsdb.models_dir(), w)
launch_eval_job(b_model_path + ".pb", w_model_path + ".pb",
"{:d}-{:d}".format(black_num, white_num), bucket)
def model_id(name_or_num):
db = sqlite3.connect("ratings.db")
bucket = fsdb.models_dir()
if not isinstance(name_or_num, str):
name_or_num = fsdb.get_model(name_or_num)
return rowid_for(db, bucket, name_or_num)