proc.kill()
# Sometimes the workers need extra help...
os.system('pkill -f selfplay_worker.py')
sys.stdout.flush()
if sys.argv[3] == 'clean_backup':
print('cleaning up {}'.format(SELFPLAY_BACKUP_DIR))
os.system('rm {}/*'.format(SELFPLAY_BACKUP_DIR))
qmeas.stop_time('selfplay_wait')
if __name__ == '__main__':
#tf.logging.set_verbosity(tf.logging.INFO)
qmeas.start(os.path.join(BASE_DIR, 'stats'))
# make sure seed is small enough
SEED = int(sys.argv[1]) % 65536
# make sure ITERATION is small enough because we use nanosecond as ITERATION
ITERATION = int(sys.argv[2]) % 65536
# get TF logger
log = logging.getLogger('tensorflow')
log.setLevel(logging.DEBUG)
# create formatter and add it to the handlers
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# create file handler which logs even debug messages
bury_latest_model()
if __name__ == '__main__':
#tf.logging.set_verbosity(tf.logging.INFO)
seed = int(sys.argv[1])
iteration = int(sys.argv[2])
print('Setting random seed, iteration = ', seed, iteration)
seed = hash(seed) + iteration
print("training seed: ", seed)
random.seed(seed)
tf.set_random_seed(seed)
numpy.random.seed(seed)
qmeas.start(os.path.join(BASE_DIR, 'stats'))
# get TF logger
log = logging.getLogger('tensorflow')
log.setLevel(logging.DEBUG)
# create formatter and add it to the handlers
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# create file handler which logs even debug messages
fh = logging.FileHandler('tensorflow.log')
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
log.addHandler(fh)
rl_loop()
qmeas.end()
print("Gathering game output...")
qmeas.start_time('main-loop-gather')
main.gather(input_directory=selfplay_dir,
output_directory=gather_dir)
qmeas.stop_time('main-loop-gather')
print("Training on gathered game data...")
qmeas.start_time('main-loop-train')
main.train(working_dir,
gather_dir,
next_model_save_file,
generation_num=1)
qmeas.stop_time('main-loop-train')
print("Trying validate on 'holdout' game...")
qmeas.start_time('main-loop-validate')
main.validate(working_dir, holdout_dir)
qmeas.stop_time('main-loop-validate')
print("Verifying that new checkpoint is playable...")
main.selfplay(load_file=next_model_save_file,
holdout_dir=holdout_dir,
output_dir=model_selfplay_dir,
output_sgf=sgf_dir,
readouts=10)
qmeas.stop_time('main-loop')
qmeas._flush()
if __name__ == '__main__':
qmeas.start()
rl_loop()
qmeas.end()
print("See sgf files here?")
sgf_listing = subprocess.check_output(["ls", "-l", sgf_dir + "/full"])
print(sgf_listing.decode("utf-8"))
print("Gathering game output...")
qmeas.start_time('main-loop-gather')
main.gather(input_directory=selfplay_dir, output_directory=gather_dir)
qmeas.stop_time('main-loop-gather')
print("Training on gathered game data...")
qmeas.start_time('main-loop-train')
main.train(working_dir, gather_dir, next_model_save_file, generation_num=1)
qmeas.stop_time('main-loop-train')
print("Trying validate on 'holdout' game...")
qmeas.start_time('main-loop-validate')
main.validate(working_dir, holdout_dir)
qmeas.stop_time('main-loop-validate')
print("Verifying that new checkpoint is playable...")
main.selfplay(
load_file=next_model_save_file,
holdout_dir=holdout_dir,
output_dir=model_selfplay_dir,
output_sgf=sgf_dir,
readouts=10)
qmeas.stop_time('main-loop')
qmeas._flush()
if __name__ == '__main__':
qmeas.start()
rl_loop()
qmeas.end()
