proc.kill() # Sometimes the workers need extra help... os.system('pkill -f selfplay_worker.py') sys.stdout.flush() if sys.argv[3] == 'clean_backup': print('cleaning up {}'.format(SELFPLAY_BACKUP_DIR)) os.system('rm {}/*'.format(SELFPLAY_BACKUP_DIR)) qmeas.stop_time('selfplay_wait') if __name__ == '__main__': #tf.logging.set_verbosity(tf.logging.INFO) qmeas.start(os.path.join(BASE_DIR, 'stats')) # make sure seed is small enough SEED = int(sys.argv[1]) % 65536 # make sure ITERATION is small enough because we use nanosecond as ITERATION ITERATION = int(sys.argv[2]) % 65536 # get TF logger log = logging.getLogger('tensorflow') log.setLevel(logging.DEBUG) # create formatter and add it to the handlers formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') # create file handler which logs even debug messages
bury_latest_model() if __name__ == '__main__': #tf.logging.set_verbosity(tf.logging.INFO) seed = int(sys.argv[1]) iteration = int(sys.argv[2]) print('Setting random seed, iteration = ', seed, iteration) seed = hash(seed) + iteration print("training seed: ", seed) random.seed(seed) tf.set_random_seed(seed) numpy.random.seed(seed) qmeas.start(os.path.join(BASE_DIR, 'stats')) # get TF logger log = logging.getLogger('tensorflow') log.setLevel(logging.DEBUG) # create formatter and add it to the handlers formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') # create file handler which logs even debug messages fh = logging.FileHandler('tensorflow.log') fh.setLevel(logging.DEBUG) fh.setFormatter(formatter) log.addHandler(fh) rl_loop() qmeas.end()
print("Gathering game output...") qmeas.start_time('main-loop-gather') main.gather(input_directory=selfplay_dir, output_directory=gather_dir) qmeas.stop_time('main-loop-gather') print("Training on gathered game data...") qmeas.start_time('main-loop-train') main.train(working_dir, gather_dir, next_model_save_file, generation_num=1) qmeas.stop_time('main-loop-train') print("Trying validate on 'holdout' game...") qmeas.start_time('main-loop-validate') main.validate(working_dir, holdout_dir) qmeas.stop_time('main-loop-validate') print("Verifying that new checkpoint is playable...") main.selfplay(load_file=next_model_save_file, holdout_dir=holdout_dir, output_dir=model_selfplay_dir, output_sgf=sgf_dir, readouts=10) qmeas.stop_time('main-loop') qmeas._flush() if __name__ == '__main__': qmeas.start() rl_loop() qmeas.end()
print("See sgf files here?") sgf_listing = subprocess.check_output(["ls", "-l", sgf_dir + "/full"]) print(sgf_listing.decode("utf-8")) print("Gathering game output...") qmeas.start_time('main-loop-gather') main.gather(input_directory=selfplay_dir, output_directory=gather_dir) qmeas.stop_time('main-loop-gather') print("Training on gathered game data...") qmeas.start_time('main-loop-train') main.train(working_dir, gather_dir, next_model_save_file, generation_num=1) qmeas.stop_time('main-loop-train') print("Trying validate on 'holdout' game...") qmeas.start_time('main-loop-validate') main.validate(working_dir, holdout_dir) qmeas.stop_time('main-loop-validate') print("Verifying that new checkpoint is playable...") main.selfplay( load_file=next_model_save_file, holdout_dir=holdout_dir, output_dir=model_selfplay_dir, output_sgf=sgf_dir, readouts=10) qmeas.stop_time('main-loop') qmeas._flush() if __name__ == '__main__': qmeas.start() rl_loop() qmeas.end()