def start(config: Config):
set_session_config(per_process_gpu_memory_fraction=1,
allow_growth=True,
device_list=config.opts.device_list)
m = Manager()
while True:
model_bt = load_model(config, config.resource.model_best_config_path,
config.resource.model_best_weight_path)
modelbt_pipes = m.list([
model_bt.get_pipes(need_reload=False)
for _ in range(config.play.max_processes)
])
model_ng = load_model(config,
config.resource.next_generation_config_path,
config.resource.next_generation_weight_path)
while not model_ng:
logger.info(f"Next generation model is None, wait for 300s")
sleep(300)
model_ng = load_model(config,
config.resource.next_generation_config_path,
config.resource.next_generation_weight_path)
logger.info(f"Next generation model has loaded!")
modelng_pipes = m.list([
model_ng.get_pipes(need_reload=False)
for _ in range(config.play.max_processes)
])
# play_worker = EvaluateWorker(config, model1_pipes, model2_pipes)
# play_worker.start()
with ProcessPoolExecutor(
max_workers=config.play.max_processes) as executor:
futures = []
for i in range(config.play.max_processes):
eval_worker = EvaluateWorker(config,
modelbt_pipes,
modelng_pipes,
pid=i)
futures.append(executor.submit(eval_worker.start))
wait(futures)
model_bt.close_pipes()
model_ng.close_pipes()
# compute whether to update best model
# and remove next generation model
score = 0
for future in futures:
score += future.result()
game_num = config.eval.game_num * config.play.max_processes
logger.info(f"Evaluate over, next generation win {score}/{game_num}")
if score * 1.0 / game_num >= config.eval.next_generation_replace_rate:
logger.info("Best model will be replaced by next generation model")
replace_best_model(config)
else:
logger.info(
"Next generation fail to defeat best model and will be removed"
)
remove_ng_model(config)
def start(config: Config):
set_session_config(per_process_gpu_memory_fraction=1,
allow_growth=True,
device_list=config.opts.device_list)
m = Manager()
response = http_request(config.internet.get_evaluate_model_url)
while int(response['status']) == 0:
data = response['data']
logger.info(
f"评测开始，基准模型：{data['base']['digest'][0:8]}, elo = {data['base']['elo']};"
f"待评测模型：{data['unchecked']['digest'][0:8]}, elo = {data['unchecked']['elo']}"
)
# make path
base_weight_path = os.path.join(
config.resource.next_generation_model_dir,
data['base']['digest'] + '.h5')
ng_weight_path = os.path.join(
config.resource.next_generation_model_dir,
data['unchecked']['digest'] + '.h5')
# load model
model_base, hist_base = load_model(config, base_weight_path,
data['base']['digest'])
model_ng, hist_ng = load_model(config, ng_weight_path,
data['unchecked']['digest'])
# make pipes
model_base_pipes = m.list([
model_base.get_pipes(need_reload=False)
for _ in range(config.play.max_processes)
])
model_ng_pipes = m.list([
model_ng.get_pipes(need_reload=False)
for _ in range(config.play.max_processes)
])
# eval_worker = EvaluateWorker(config, model_base_pipes, model_ng_pipes, 0, data)
# res = eval_worker.start()
with ProcessPoolExecutor(
max_workers=config.play.max_processes) as executor:
futures = []
for i in range(config.play.max_processes):
eval_worker = EvaluateWorker(config, model_base_pipes,
model_ng_pipes, i, data,
hist_base, hist_ng)
futures.append(executor.submit(eval_worker.start))
sleep(1)
wait(futures)
# close pipe
model_base.close_pipes()
model_ng.close_pipes()
# reset model
model_base = None
model_ng = None
response = http_request(config.internet.get_evaluate_model_url)
logger.info(f"没有待评测权重，请稍等或继续跑谱")
def start(config: Config):
set_session_config(per_process_gpu_memory_fraction=1, allow_growth=True, device_list=config.opts.device_list)
current_model = load_model(config)
m = Manager()
cur_pipes = m.list([current_model.get_pipes() for _ in range(config.play.max_processes)])
# play_worker = SelfPlayWorker(config, cur_pipes, 0)
# play_worker.start()
with ProcessPoolExecutor(max_workers=config.play.max_processes) as executor:
futures = []
for i in range(config.play.max_processes):
play_worker = SelfPlayWorker(config, cur_pipes, i)
logger.debug("Initialize selfplay worker")
futures.append(executor.submit(play_worker.start))
def start(config: Config):
set_session_config(per_process_gpu_memory_fraction=1,
allow_growth=True,
device_list=config.opts.device_list)
return EvaluateWorker(config).start()