def train(working_dir):
model_num, model_name = fsdb.get_latest_model()
print("Training on gathered game data, initializing from {}".format(model_name))
new_model_num = model_num + 1
new_model_name = shipname.generate(new_model_num)
print("New model will be {}".format(new_model_name))
training_file = os.path.join(
fsdb.golden_chunk_dir(), str(new_model_num) + '.tfrecord.zz')
while not gfile.Exists(training_file):
print("Waiting for", training_file)
time.sleep(1*60)
print("Using Golden File:", training_file)
try:
save_file = os.path.join(fsdb.models_dir(), new_model_name)
print("Training model")
dual_net.train(training_file)
print("Exporting model to ", save_file)
dual_net.export_model(working_dir, save_file)
except Exception as e:
import traceback
logging.error(traceback.format_exc())
print(traceback.format_exc())
logging.exception("Train error")
sys.exit(1)
def train(working_dir):
model_num, model_name = fsdb.get_latest_model()
games = gfile.Glob(os.path.join(fsdb.selfplay_dir(), model_name, '*.zz'))
if len(games) < MIN_GAMES_PER_GENERATION:
print("{} doesn't have enough games to train a new model yet ({})".
format(model_name, len(games)))
print("Sleeping...")
time.sleep(10 * 60)
print("Done...")
sys.exit(1)
print("Training on gathered game data, initializing from {}".format(
model_name))
new_model_num = model_num + 1
new_model_name = shipname.generate(new_model_num)
print("New model will be {}".format(new_model_name))
training_file = os.path.join(fsdb.golden_chunk_dir(),
str(new_model_num) + '.tfrecord.zz')
while not gfile.Exists(training_file):
print("Waiting for", training_file)
time.sleep(1 * 60)
print("Using Golden File:", training_file)
save_file = os.path.join(fsdb.models_dir(), new_model_name)
try:
main.train(working_dir, [training_file],
save_file,
generation_num=model_num + 1)
except:
logging.exception("Train error")
def train(load_dir=MODELS_DIR, save_dir=MODELS_DIR):
model_num, model_name = get_latest_model()
games = gfile.Glob(os.path.join(SELFPLAY_DIR, model_name, '*.zz'))
if len(games) < MIN_GAMES_PER_GENERATION:
print("{} doesn't have enough games to train a new model yet ({})".
format(model_name, len(games)))
print("Sleeping...")
time.sleep(10 * 60)
print("Done...")
sys.exit(1)
print("Training on gathered game data, initializing from {}".format(
model_name))
new_model_num = model_num + 1
new_model_name = shipname.generate(new_model_num)
print("New model will be {}".format(new_model_name))
training_file = os.path.join(GOLDEN_CHUNK_DIR,
str(new_model_num) + '.tfrecord.zz')
while not gfile.Exists(training_file):
print("Waiting for", training_file)
time.sleep(1 * 60)
print("Using Golden File:", training_file)
load_file = os.path.join(load_dir, model_name)
save_file = os.path.join(save_dir, new_model_name)
try:
main.train(ESTIMATOR_WORKING_DIR, [training_file],
save_file,
generation_num=model_num + 1)
except:
logging.exception("Train error")
def train(logdir=None):
model_num, model_name = get_latest_model()
print("Training on gathered game data, initializing from {}".format(model_name))
new_model_name = shipname.generate(model_num + 1)
print("New model will be {}".format(new_model_name))
load_file = os.path.join(MODELS_DIR, model_name)
save_file = os.path.join(MODELS_DIR, new_model_name)
main.train(TRAINING_CHUNK_DIR, save_file=save_file, load_file=load_file,
generation_num=model_num, logdir=logdir)
def train():
model_num, model_name = get_latest_model()
print("Training on gathered game data, initializing from {}".format(
model_name))
new_model_name = shipname.generate(model_num + 1)
print("New model will be {}".format(new_model_name))
load_file = os.path.join(MODELS_DIR, model_name)
save_file = os.path.join(MODELS_DIR, new_model_name)
#try:
main.train(ESTIMATOR_WORKING_DIR,
TRAINING_CHUNK_DIR,
save_file,
generation_num=model_num + 1)
def train():
model_num, model_name = get_latest_model()
print("Training on gathered game data, initializing from {}".format(model_name))
new_model_name = shipname.generate(model_num + 1)
print("New model will be {}".format(new_model_name))
load_file = os.path.join(MODELS_DIR, model_name)
save_file = os.path.join(MODELS_DIR, new_model_name)
try:
main.train(ESTIMATOR_WORKING_DIR, TRAINING_CHUNK_DIR, save_file,
generation_num=model_num + 1)
except:
print("Got an error training, muddling on...")
logging.exception("Train error")
def bootstrap(working_dir):
bootstrap_name = shipname.generate(0)
bootstrap_model_path = os.path.join(fsdb.models_dir(), bootstrap_name)
print("Bootstrapping with working dir {}\n Model 0 exported to {}".format(
working_dir, bootstrap_model_path))
main.bootstrap(working_dir, bootstrap_model_path)
def bootstrap():
bootstrap_name = shipname.generate(0)
bootstrap_model_path = os.path.join(MODELS_DIR, bootstrap_name)
print("Bootstrapping model at {}".format(bootstrap_model_path))
main.bootstrap(bootstrap_model_path)
def bootstrap():
bootstrap_name = shipname.generate(0)
bootstrap_model_path = os.path.join(MODELS_DIR, bootstrap_name)
print("Bootstrapping with working dir {}\n Model 0 exported to {}".format(
ESTIMATOR_WORKING_DIR, bootstrap_model_path))
main.bootstrap(ESTIMATOR_WORKING_DIR, bootstrap_model_path)
def bootstrap():
bootstrap_name = shipname.generate(0)
bootstrap_model_path = os.path.join(MODELS_DIR, bootstrap_name)
print("Bootstrapping with working dir {}\n Model 0 exported to {}".format(
ESTIMATOR_WORKING_DIR, bootstrap_model_path))
main.bootstrap(ESTIMATOR_WORKING_DIR, bootstrap_model_path)
def test_bootstrap_gen(self):
name = shipname.generate(0)
self.assertIn('bootstrap', name)
def bootstrap():
bootstrap_name = shipname.generate(0)
bootstrap_model_path = os.path.join(MODELS_DIR, bootstrap_name)
print("Bootstrapping model at {}".format(bootstrap_model_path))
main.bootstrap(bootstrap_model_path)
def test_bootstrap_gen(self):
name = shipname.generate(0)
self.assertIn('bootstrap', name)
