def test_save_sgf(self):
model = DummyModel()
init_simulation_workers()
mcts_simulations = 8 # mcts batch size is 8 and we need at least one batch
game_data = play_game(model,
model,
mcts_simulations,
conf['STOP_EXPLORATION'],
self_play=True,
num_moves=10)
save_game_sgf("test_model", 0, game_data)
destroy_simulation_workers()
os.remove("games/test_model/game_000.sgf")
os.removedirs("games/test_model")
def run(self):
# set environment
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = str(self._gpuid)
logger.info('cuda_visible_device %s',
os.environ["CUDA_VISIBLE_DEVICES"])
if conf['THREAD_SIMULATION']:
init_simulation_workers()
# load models
latest_model, best_model = self.load_model()
while True:
if latest_model.name != best_model.name:
total = 0
wins = 0
desc = "Evaluation %s vs %s" % (latest_model.name,
best_model.name)
tq = tqdm(range(EVALUATE_N_GAMES), desc=desc)
for game in tq:
if self._one_game_only >= 0 and tq != game:
continue
directory = os.path.join(EVAL_DIR, latest_model.name,
"game_%03d" % game)
if os.path.isdir(directory):
continue
try:
os.makedirs(directory)
except Exception:
continue
start = datetime.datetime.now()
game_data = play_game(best_model,
latest_model,
MCTS_SIMULATIONS,
stop_exploration=0)
stop = datetime.datetime.now()
# Some statistics
winner_model = game_data['winner_model']
if winner_model == latest_model.name:
wins += 1
total += 1
moves = len(game_data['moves'])
new_desc = desc + " (winrate:%s%% %.2fs/move)" % (int(
wins / total * 100), (stop - start).seconds / moves)
tq.set_description(new_desc)
# save_game_data(best_model.name, game, game_data)
self.save_eval_game(latest_model.name, game, winner_model)
if self._one_game_only >= 0:
break
else:
logger.info("No new trained model")
if self._forever:
logger.info("Sleep for %s seconds", conf['SLEEP_SECONDS'])
time.sleep(conf['SLEEP_SECONDS'])
if not self._forever:
break
latest_model, best_model = self.load_model()
destroy_simulation_workers()
def run(self):
if self._task == "promote_best_model":
promote_best_model()
return
try:
init_simulation_workers_by_gpuid(self._gpuid)
best_model_name = put_name_request("BEST_NAME")
latest_model_name = put_name_request("LATEST_NAME")
if latest_model_name != best_model_name:
total = 0
wins = 0
desc = "Evaluation %s vs %s for %s games" % (
latest_model_name, best_model_name, EVALUATE_N_GAMES)
tq = tqdm(range(EVALUATE_N_GAMES), desc=desc)
for game in tq:
directory = os.path.join(EVAL_DIR, latest_model_name,
"game_%03d" % game)
if os.path.isdir(directory):
continue
try:
os.makedirs(directory)
except Exception:
continue
start = datetime.datetime.now()
game_data = play_game_async("BEST_SYM",
"LATEST_SYM",
conf['ENERGY'],
stop_exploration=0,
process_id=self._gpuid)
stop = datetime.datetime.now()
# Some statistics
winner_model = game_data['winner_model']
if winner_model == latest_model_name:
wins += 1
total += 1
moves = len(game_data['moves'])
new_desc = desc + " (winrate:%s%% %.2fs/move)" % (int(
wins / total * 100), (stop - start).seconds / moves)
tq.set_description(new_desc)
game_name = "eval_game_%03d" % game
save_game_data(
latest_model_name,
game,
game_data,
game_name="eval_game"
) # save game data to self-play folder for training
self.save_eval_game(
latest_model_name, game,
winner_model) # save game result for statistic
if conf['TRAINING_SERVER']:
# Copy eval_game to self-play dir in training server
sync_eval_game_data(latest_model_name, game_name)
else:
print("BEST MODEL and LAST MODEL are the same!! Quitting")
destroy_simulation_workers()
except Exception as e:
print("EXCEPTION IN NO MODEL EVALUATION WORKER!!!: %s" % e)
import traceback
traceback.print_exc()
def tearDown(self):
destroy_simulation_workers()