def linear_approximator_compete():
policyA = PolicyValueNet(width, height, model_file = dnn_file)
playerA = MCTSPlayer(policyA.policy_value_fn, c_puct=5, n_playout=400, save_tree_on_compete= False)
policyB = PolicyValueNet(width, height, PolicyValueNetCls= LinearNet, model_file = lin_file)
playerB = MCTSPlayer(policyB.policy_value_fn, c_puct=5, n_playout=400, save_tree_on_compete= False)
run(playerA, playerB)
def improvement_compete():
policyA = PolicyValueNet(width, height, model_file = dnn_file)
playerA = MCTSPlayer(policyA.policy_value_fn, c_puct=5, n_playout=400, save_tree_on_compete= True)
policyB = PolicyValueNet(width, height, model_file = dnn_file)
playerB = MCTSPlayer(policyB.policy_value_fn, c_puct=5, n_playout=400, save_tree_on_compete= False)
run(playerA, playerB)
def __init__(self, init_model=None):
# params of the board and the game
self.writer = SummaryWriter()
self.board_width = BOARD_SIZE
self.board_height = BOARD_SIZE
self.n_resnet = N_RESNET
self.in_channel = IN_CHANNEL
self.n_in_row = N_ROW
self.board = Board(width=self.board_width,
height=self.board_height,
n_in_row=self.n_in_row)
self.game = Game(self.board,
state_representation_channel=self.in_channel)
# training params
self.learn_rate = 2e-3
self.lr_multiplier = 1.0 # adaptively adjust the learning rate based on KL
self.temp = 1.0 # the temperature param
self.n_playout = 400 # num of simulations for each move
self.c_puct = 5
self.buffer_size = 10000
self.batch_size = 512 # mini-batch size for training
self.data_buffer = deque(maxlen=self.buffer_size)
self.play_batch_size = 1
self.epochs = 5 # num of train_steps for each update
self.kl_targ = 0.02
self.check_freq = SAVE_FREQ
self.game_batch_num = TRAIN_EPOCH
self.best_win_ratio = 0.0
self.best_policy = 0
# num of simulations used for the pure mcts, which is used as
# the opponent to evaluate the trained policy
self.pure_mcts_playout_num = 1000
if init_model:
# start training from an initial policy-value net
self.policy_value_net = PolicyValueNet(self.board_width,
self.board_height,
self.n_resnet,
self.in_channel,
model_file=init_model,
use_gpu=USE_GPU)
else:
# start training from a new policy-value net
self.policy_value_net = PolicyValueNet(self.board_width,
self.board_height,
self.n_resnet,
self.in_channel,
use_gpu=USE_GPU)
self.mcts_player = MCTSPlayer(self.policy_value_net.policy_value_fn,
c_puct=self.c_puct,
n_playout=self.n_playout,
is_selfplay=1)
def run():
n = N
width, height = SIZE,SIZE
# if MCTS_PURE:
# player_1 = MCTS_Pure(c_puct=5, n_playout=PLAYOUT)
# print ("Benchmarking the following two models:"+MODEL_1+" Pure MCTS")
# elif HUMAN:
# player_2=Human()
# print ("Benchmarking the following two models:"+MODEL_1+" Human")
# else:
# print ("Benchmarking the following two models:"+MODEL_1+" vs "+MODEL_2)
# policy_2= PolicyValueNet(width, height, model_file=MODEL_2,state_representation_channel = 4)
# player_2 = MCTSPlayer(policy_2.policy_value_fn,c_puct=5,n_playout=400) # set larger n_playout for better performance
#
policy_1= PolicyValueNet(width, height, model_file=MODEL_1,in_channel = 11,n_resnet=1)
player_1 = MCTSPlayer(policy_1.policy_value_fn,
c_puct=5,
n_playout=400) # set larger n_playout for better performance
# player_1 = Human()
player_2 = Human()
win_ratio = policy_evaluate(player_1,player_2)
print("The win ratio for "+MODEL_1+" is: ",str(100*win_ratio)+"%")
def run():
# n = 5
width, height = 9, 9 # width, height = 8, 8
# model_file = './best_policy.pkl'
# board = Board(width=width, height=height, n_in_row=n)
board = Board(width=width, height=height)
game = Game(board)
# ############### human VS AI ###################
# load the trained policy_value_net in either Theano/Lasagne, PyTorch or TensorFlow
# best_policy = PolicyValueNet(width, height, model_file = model_file)
# mcts_player = MCTSPlayer(best_policy.policy_value_fn, c_puct=5, n_playout=400)
# load the provided model (trained in Theano/Lasagne) into a MCTS player written in pure numpy
# try:
# policy_param = pickle.load(open(model_file, 'rb'))
# policy_param = pickle.load(open(model_file, 'rb'),
# encoding='bytes') # To support python3
best_policy = PolicyValueNet(width, height)
mcts_player = MCTSPlayer(
best_policy.policy_value_fn, c_puct=5,
n_playout=400) # set larger n_playout for better performance
# uncomment the following line to play with pure MCTS (it's much weaker even with a larger n_playout)
# mcts_player = MCTS_Pure(c_puct=5, n_playout=1000)
# human player, input your move in the format: 2,3
human = Human()
# set start_player=0 for human first
game.start_play(human, mcts_player, start_player=0, is_shown=1)
def run_game(surface, omok, menu):
omok.turn = black_stone
omok.init_game()
board = Board(width=board_size, height=board_size, n_in_row=5)
game = Game(board)
model_file = 'current_policy_15x15-self500.model'
while True:
for event in pygame.event.get():
pygame.display.flip()
if omok.turn == black_stone and event.type == MOUSEBUTTONUP:
x, y = omok.check_board_black(event.pos)
human = Human(x, y)
elif omok.turn == white_stone:
best_policy = PolicyValueNet(15, 15, model_file=model_file)
mcts = MCTSPlayer(best_policy.policy_value_fn,
c_puct=7,
n_playout=800)
move = game.start_play(human, mcts, start_player=1)
x = move / board_size
y = move % board_size
print(int(x), y)
omok.check_board_white(int(x), y)
else:
pass
if omok.is_gameover:
return
pygame.display.update()
fps_clock.tick(fps)
def loadAI(self, init_model):
""""
# load AI
"""
# return
# self.buffer_size = 10000
# self.batch_size = 512 # mini-batch size for training
# self.data_buffer = deque(maxlen=self.buffer_size)
init_model = "current_policy.model"
init_model = "best_policy.model"
init_model = 'best_policy_12000.pt'
# init_model = 'best_policy200.pt'
self.result = False
# self.policy_value_net = PolicyValueNet(self.width,
# self.height,
# model_file=False)
self.policy_value_net = PolicyValueNet(self.width,
self.height,
model_file=init_model,
use_gpu=False)
self.mcts_player = MCTSPlayer(self.policy_value_net.policy_value_fn,
c_puct=self.c_puct,
n_playout=self.n_playout,
is_selfplay=0)
self.board2 = Board2(width=self.width,
height=self.height,
n_in_row=self.n_in_row)
self.board2.init_board(1)
self.game = Game(self.board2)
def run():
# n = 5
width, height = 9, 9 # width, height = 8, 8
model_file = r'./current_policy_cpu.pkl'
try:
board = Board(width=width, height=height)
game = Game(board)
# ############### human VS AI ###################
# load the trained policy_value_net in either Theano/Lasagne, PyTorch or TensorFlow
best_policy = PolicyValueNet(width, height, model_file = model_file)
# mcts_player = MCTSPlayer(best_policy.policy_value_fn, c_puct=5, n_playout=400)
# load the provided model (trained in Theano/Lasagne) into a MCTS player written in pure numpy
# best_policy = PolicyValueNetNumpy(width, height, policy_param)
mcts_player = MCTSPlayer(best_policy.policy_value_fn,
c_puct=5,
n_playout=400) # set larger n_playout for better performance
# uncomment the following line to play with pure MCTS (it's much weaker even with a larger n_playout)
# mcts_player = MCTS_Pure(c_puct=5, n_playout=1000)
# human player, input your move in the format: 2,3
human = Human()
# set start_player=0 for human first
game.start_play(human, mcts_player, start_player=0, is_shown=1)
except KeyboardInterrupt:
print('\n\rquit')
def run():
n = 5
width, height = 15, 15
model_file = 'best_policy_15_15_5_forbidden_pyt.model'
try:
board = Board(width=width, height=height, n_in_row=n, forbidden_check_level=-1)
game = Game(board)
# ############### human VS AI ###################
# load the trained policy_value_net in either Theano/Lasagne, PyTorch or TensorFlow
best_policy = PolicyValueNet(width, height, model_file = model_file)
mcts_player = MCTSPlayer(best_policy.policy_value_fn, c_puct=5, n_playout=400)
# load the provided model (trained in Theano/Lasagne) into a MCTS player written in pure numpy
try:
policy_param = pickle.load(open(model_file, 'rb'))
except:
policy_param = pickle.load(open(model_file, 'rb'),
encoding='bytes') # To support python3
# best_policy = PolicyValueNetNumpy(width, height, policy_param)
# mcts_player = MCTSPlayer(best_policy.policy_value_fn,
# c_puct=5,
# n_playout=400) # set larger n_playout for better performance
# uncomment the following line to play with pure MCTS (it's much weaker even with a larger n_playout)
# mcts_player = MCTS_Pure(c_puct=5, n_playout=1000)
# human player, input your move in the format: 2,3
human = Human()
# set start_player=0 for human first
game.start_play(human, mcts_player, start_player=0, is_shown=1)
except KeyboardInterrupt:
print('\n\rquit')
def run():
n = 5
# 这里可以修改棋盘的大小，需要和AI Model的棋盘大小相等
width, height = 10, 10
# 调用AI模型
model_file = 'best_policy.model'
try:
# 初始化棋盘
board = Board(width=width, height=height, n_in_row=n)
game = Game(board)
# ############### human VS AI ###################
# 加载AI Model
best_policy = PolicyValueNet(width,
height,
model_file=model_file,
use_gpu=False)
# 设置n_playout越大，效果越好，不需要设置is_selfplay，因为不需要进行AI训练
mcts_player = MCTSPlayer(best_policy.policy_value_fn,
c_puct=5,
n_playout=400)
# 也可以使用MCTS_Pure进行对弈，但是它太弱了
# mcts_player = MCTS_Pure(c_puct=5, n_playout=1000)
# 创建人类player, 输入下棋位置比如 3,3
human = Human()
# start_player=1表示电脑先手，0表示人先手
game.start_play(human, mcts_player, start_player=1, is_shown=1)
except KeyboardInterrupt:
print('\n\rquit')
def run(
n_in_row,
width,
height, # 几子棋，棋盘宽度，高度
model_file,
ai_first, # 载入的模型文件，是否AI先下棋
n_playout,
use_gpu): # AI每次进行蒙特卡洛的模拟次数，是否使用GPU
try:
board = Board(width=width, height=height, n_in_row=n_in_row) # 产生一个棋盘
# ############### human VS AI ###################
best_policy = PolicyValueNet(width,
height,
model_file=model_file,
use_gpu=use_gpu) # 加载最佳策略网络
mcts_player = MCTSPlayer(best_policy.policy_value_fn,
c_puct=5,
n_playout=n_playout) # 生成一个AI玩家
main = UserInterface_GO_Human_vs_AI(
mcts_player,
board,
width,
height,
)
main.test()
# set start_player=0 for human first
# game.start_play(human, mcts_player, start_player=ai_first, is_shown=1) # 开始游戏
except KeyboardInterrupt:
print('\n\rquit')
def __init__(self, init_model=None):
# params of the board and the game
self.board_width = 6
self.board_height = 6
self.n_in_row = 4
self.board = Board(width=self.board_width,
height=self.board_height,
n_in_row=self.n_in_row)
self.game = Game(self.board)
# training params
self.learn_rate = 2e-3
self.data_buffer = deque(maxlen=1000)
self.batch_size = 10
self.temp = 1.0 # the temperature param
self.n_playout = 40 # num of simulations for each move
self.c_puct = 5
self.epochs = 50
self.pure_mcts_playout_num = 2
self.best_win_ratio = 0.0
self.policy_value_net = PolicyValueNet(self.board_width,
self.board_height)
self.mcts_player = MCTSPlayer(self.policy_value_net.policy_value_fn,
c_puct=self.c_puct,
n_playout=self.n_playout,
is_selfplay=1)
def __init__(self, config: Config):
# params of the game
self.config = config
self.game = Game.from_config(config)
# training params
self.buffer_size = 10000
self.min_data_to_collect = 128
self.batch_size = 128 # mini-batch size for training
self.data_buffer = deque(maxlen=self.buffer_size)
self.save_freq = 20
self.eval_freq = self.save_freq * 100
self.num_total_iter = self.eval_freq * 4
assert self.num_total_iter % self.save_freq == 0
assert self.num_total_iter % self.eval_freq == 0
self.best_win_ratio = 0.0
# num of simulations used for the pure mcts, which is used as
# the opponent to evaluate the trained policy
self.policy_value_net = PolicyValueNet(self.config.size,
model_file=config.model_file)
self.mcts_player = MCTSPlayer(
self.policy_value_net,
c_puct=config.c_puct,
n_playout=config.n_playout,
is_selfplay=True,
)
def __init__(self, init_model=None):
# params of the board and the game
self.board_width = 8 #6
self.board_height = 8 #6
self.n_in_row = 5 #4
self.board = Board(width=self.board_width,
height=self.board_height,
n_in_row=self.n_in_row)
self.game = Game(self.board)
# training params
self.learn_rate = 5e-3
self.lr_multiplier = 1.0 # adaptively adjust the learning rate based on KL
self.temp = 1.0 # the temperature param
self.n_playout = 400 # num of simulations for each move
# c_puct是MCTS里用来控制exploration-exploit tradeoff的参数
# 这个参数越大的话MCTS搜索的过程中就偏向于均匀的探索，越小的话就偏向于直接选择访问次数多的分支
self.c_puct = 5
self.buffer_size = 10000
self.batch_size = 512 # mini-batch size for training
self.data_buffer = deque(maxlen=self.buffer_size)
self.play_batch_size = 1
self.epochs = 5 # num of train_steps for each update
self.kl_targ = 0.025
# 检查当前策咯胜率的频率，当前设置为每50次训练后通过自我对弈评价当前策略
# 如果找到更优策略，则保存当前策咯模型
self.check_freq = 50
#训练迭代次数
self.game_batch_num = 1500
self.best_win_ratio = 0.0
# num of simulations used for the pure mcts, which is used as the opponent to evaluate the trained policy
#每次训练蒙特卡洛树搜索的次数，初始化为1000（后续训练过程中会不断增加）
self.pure_mcts_playout_num = 1000
if init_model:
# start training from an initial policy-value net
policy_param = pickle.load(open(init_model, 'rb'))
self.policy_value_net = PolicyValueNet(self.board_width,
self.board_height,
net_params=policy_param)
else:
# start training from a new policy-value net
self.policy_value_net = PolicyValueNet(self.board_width,
self.board_height)
self.mcts_player = MCTSPlayer(self.policy_value_net.policy_value_fn,
c_puct=self.c_puct,
n_playout=self.n_playout,
is_selfplay=1)
def run():
n = 6
width, height = 9, 9
model_file = 'best_policy.model'
model_file_compare = 'compare.model'
try:
board = Board(width=width, height=height, n_in_row=n)
game = Game(board)
try:
policy_param = pickle.load(open(model_file, 'rb'))
policy_param_compare = pickle.load(open(model_file_compare, 'rb'))
except:
policy_param = pickle.load(open(model_file, 'rb'),
encoding='bytes')
policy_param_compare = pickle.load(
open(model_file_compare,
'rb'), encoding='bytes') # To support python3
#對照組
best_policy = PolicyValueNet(width, height, policy_param)
mcts_player = MCTSPlayer(
best_policy.policy_value_fn, c_puct=5,
n_playout=800) # set larger n_playout for better performance
#實驗組
compare = PolicyValueNet(width, height, policy_param_compare)
mcts_player_compare = MCTSPlayer(
compare.policy_value_fn, c_puct=5,
n_playout=800) # set larger n_playout for better performance
#評估勝率
win_cnt = defaultdict(int)
for i in range(200): #場數設定
winner = game.start_play(mcts_player_compare,
mcts_player,
start_player=i % 2,
is_shown=1)
win_cnt[winner] += 1
print("win: {}, lose: {}, tie:{}".format(win_cnt[1], win_cnt[2],
win_cnt[-1]))
except KeyboardInterrupt:
print('\n\rquit')
def __init__(self, init_model=None):
# params of the board and the game
self.board_width = 15
self.board_height = 15
self.n_in_row = 5
self.board = Board(width=self.board_width,
height=self.board_height,
n_in_row=self.n_in_row)
self.game = Game(self.board)
# training params
self.learn_rate = 2e-3
self.lr_multiplier = 1.0 # adaptively adjust the learning rate based on KL
self.temp = 1.0 # the temperature param
self.n_playout = 400 # num of simulations for each move
self.c_puct = 5
self.buffer_size = 10000
self.batch_size = 512 # mini-batch size for training
self.data_buffer = deque(maxlen=self.buffer_size)
self.play_batch_size = 1
self.epochs = 5 # num of train_steps for each update
self.kl_targ = 0.02
self.check_freq = 200
self.game_batch_num = 5000
self.best_win_ratio = 0.0
# num of simulations used for the pure mcts, which is used as
# the opponent to evaluate the trained policy
self.pure_mcts_playout_num = 1000
#init_model = "best_policy.pt"
if init_model:
# start training from an initial policy-value net
self.policy_value_net = PolicyValueNet(self.board_width,
self.board_height,
model_file=init_model,
use_gpu=True)
else:
# start training from a new policy-value net
self.policy_value_net = PolicyValueNet(self.board_width,
self.board_height,
use_gpu=True)
self.mcts_player = MCTSPlayer(self.policy_value_net.policy_value_fn,
c_puct=self.c_puct,
n_playout=self.n_playout,
is_selfplay=1)
def __init__(self, init_model=None):
# 设置棋盘和游戏的参数
self.board_width = 10
self.board_height = 10
self.n_in_row = 5
self.board = Board(width=self.board_width,
height=self.board_height,
n_in_row=self.n_in_row)
self.game = Game(self.board)
# 设置训练参数
self.learn_rate = 2e-3 # 基准学习率
self.lr_multiplier = 1.2 # 基于KL自动调整学习倍速
self.temp = 1.0 # 温度参数
self.n_playout = 400 # 每下一步棋，模拟的步骤数
self.c_puct = 5 # exploitation和exploration之间的折中系数
self.buffer_size = 10000
self.batch_size = 512 # mini-batch size for training
self.data_buffer = deque(maxlen=self.buffer_size) #使用 deque 创建一个双端队列
self.play_batch_size = 1
self.epochs = 5 # num of train_steps for each update
self.kl_targ = 0.02 # 早停检查
self.check_freq = 50 # 每50次检查一次，策略价值网络是否更新
self.game_batch_num = 2000 # 训练多少个epoch
self.best_win_ratio = 0.0 # 当前最佳胜率，用他来判断是否有更好的模型
# 弱AI（纯MCTS）模拟步数，用于给训练的策略AI提供对手
self.pure_mcts_playout_num = 1000
if init_model:
# 通过init_model设置策略网络
self.policy_value_net = PolicyValueNet(self.board_width,
self.board_height,
model_file=init_model,
use_gpu=True)
else:
# 训练一个新的策略网络
self.policy_value_net = PolicyValueNet(self.board_width,
self.board_height,
use_gpu=True)
# AI Player，设置is_selfplay=1 自我对弈，因为是在进行训练
self.mcts_player = MCTSPlayer(self.policy_value_net.policy_value_fn,
c_puct=self.c_puct,
n_playout=self.n_playout,
is_selfplay=1)
def __init__(self, init_model=None):
""" init function for the class"""
# params of the board and the game
self.board_width = 6 # board width
self.board_height = 6 # board height
self.n_in_row = 4 # win by n in line (vertically, horizontally, diagonally)
self.board = Board(width=self.board_width,
height=self.board_height,
n_in_row=self.n_in_row)
self.game = Game(self.board)
# training params
self.learn_rate = 2e-3
self.lr_multiplier = 1.0 # adaptively adjust the learning rate based on KL
self.temp = 1.0 # the temperature param
self.n_playout = 400 # num of simulations for each move
self.c_puct = 5 # a number in (0, inf) controlling the relative impact of value Q, and prior probability P, on this node's score.
self.buffer_size = 10000 # buffer size for replaying experience
self.batch_size = 512 # mini-batch size for training
self.data_buffer = deque(maxlen=self.buffer_size) # buffer
self.play_batch_size = 1 # size of rollout for each episode
self.epochs = 5 # num of train_steps for each update
self.kl_targ = 0.02 # target of KL loss
self.check_freq = 50 # frequency for check evaluation and save model
self.game_batch_num = 1500 # number of training game loop
self.best_win_ratio = 0.0 # best evaluated win ratio
# num of simulations used for the pure mcts, which is used as
# the opponent to evaluate the trained policy
self.pure_mcts_playout_num = 1000
if init_model: # load from existing file
# start training from an initial policy-value net
self.policy_value_net = PolicyValueNet(self.board_width,
self.board_height,
model_file=init_model)
else:
# start training from a new policy-value net
self.policy_value_net = PolicyValueNet(self.board_width,
self.board_height)
self.mcts_player = MCTSPlayer(self.policy_value_net.policy_value_fn,
c_puct=self.c_puct,
n_playout=self.n_playout,
is_selfplay=1)
def main():
config = Config.from_args()
_game = Game.from_config(config)
policy_value_net = PolicyValueNet(config.size,
model_file=config.model_file)
mcts_player = MCTSPlayer(
policy_value_net,
c_puct=config.c_puct,
n_playout=config.n_playout,
)
evaluate_policy(_game, mcts_player)
def EMD_between_two_models_on_board(model1_name,
input_plains_num_1,
i1,
model2_name,
input_plains_num_2,
i2,
board1,
board2,
width=6,
height=6,
use_gpu=True):
model_file_1 = f'/home/lirontyomkin/AlphaZero_Gomoku/models/{model1_name}/current_policy_{i1}.model'
policy_1 = PolicyValueNet(width,
height,
model_file=model_file_1,
input_plains_num=input_plains_num_1,
use_gpu=use_gpu)
model_file_2 = f'/home/lirontyomkin/AlphaZero_Gomoku/models/{model2_name}/current_policy_{i2}.model'
policy_2 = PolicyValueNet(width,
height,
model_file=model_file_2,
input_plains_num=input_plains_num_2,
use_gpu=use_gpu)
board_current_state1 = board1.current_state(last_move=True,
is_random_last_turn=False)
board_current_state2 = board2.current_state(last_move=True,
is_random_last_turn=False)
acts_policy1, probas_policy1 = zip(*policy_1.policy_value_fn(board1)[0])
acts_policy2, probas_policy2 = zip(*policy_2.policy_value_fn(board2)[0])
dist_matrix = generate_matrix_dist_metric(width)
distance = emd(np.asarray(probas_policy1, dtype='float64'),
np.asarray(probas_policy2, dtype='float64'), dist_matrix)
return distance
def __init__(self, init_model=None):
# params of the board and the game
#width of chessboard
self.board_width = 8 #6 #10
#height of chessboard
self.board_height = 8 #6 #10
#conditions for victory
self.n_in_row = 5 #4 #5
self.board = Board(width=self.board_width, height=self.board_height, n_in_row=self.n_in_row)
self.game = Game(self.board)
# training params
self.learn_rate = 5e-3 #learning rate
self.lr_multiplier = 1.0 # adaptively adjust the learning rate based on KL
self.temp = 1.0 # the temperature param
self.n_playout = 400 # num of simulations for each move
self.c_puct = 5
self.buffer_size = 10000 #The number of maximum elements in the queue
self.batch_size = 512 # mini-batch size for training
self.data_buffer = deque(maxlen=self.buffer_size) # queue size
self.play_batch_size = 1 # collect a set of data if it self-play once
self.epochs = 5 # num of train_steps for each update
self.kl_targ = 0.025 #KL target
#check frequency: evaluate the game and current AI model every 50 times of self-play
#The evaluation method is to use the latest AI model and MCTs-pure AI (based on random roll out) to fight 10 rounds
self.check_freq = 50 #50
self.game_batch_num = 200 #the number of training batches
self.best_win_ratio = 0.0 #historical best winning rate
# num of simulations used for the pure mcts, which is used as the opponent to evaluate the trained policy
self.pure_mcts_playout_num = 1000
if init_model:
# start training from an initial policy-value net
#pickle.load(file)反序列化对象。将文件中的数据解析为一个pytorch对象
policy_param = pickle.load(open(init_model, 'rb')) #使用‘rb’按照二进制位读取
self.policy_value_net = PolicyValueNet(self.board_width, self.board_height, net_params = policy_param)
else:
# start training from a new policy-value net
self.policy_value_net = PolicyValueNet(self.board_width, self.board_height)
self.mcts_player = MCTSPlayer(self.policy_value_net.policy_value_fn, c_puct=self.c_puct, n_playout=self.n_playout, is_selfplay=1)
def __init__(self,
init_model=None,
board_width=6,
board_height=6,
n_in_row=4,
n_playout=400,
use_gpu=False,
is_shown=False,
output_file_name="",
game_batch_number=1500):
# 游戏和棋盘参数
self.board_width = board_width
self.board_height = board_height
self.n_in_row = n_in_row
self.board = Board(width=self.board_width,
height=self.board_height,
n_in_row=self.n_in_row)
self.game = Game(self.board)
# 训练参数
self.learn_rate = 2e-3 #学习率α ：0.002
self.lr_multiplier = 1.0 # 根据 KL散度 适应性的调整学习率
self.temp = 1.0 # 温度参数t
self.n_playout = n_playout # 每次move的 模拟playout次数
self.c_puct = 5 #c_put常量
self.buffer_size = 10000 #缓冲区大小
self.batch_size = 512 # mini-batch size for training
self.data_buffer = deque(maxlen=self.buffer_size)
self.play_batch_size = 1
self.epochs = 5 # 每次 update 的 train_steps
self.kl_targ = 0.02
self.check_freq = 50
self.game_batch_num = game_batch_number #训练局数
self.best_win_ratio = 0.0
# 纯蒙特卡索搜索训练参数
# 目的可以是作为真正训练的模型的对手
self.pure_mcts_playout_num = 1000 #纯蒙特卡洛搜索模拟次数
self.use_gpu = use_gpu
self.is_shown = is_shown
self.output_file_name = output_file_name #输出的txt文件名
#初始化神经网络
self.policy_value_net = PolicyValueNet(self.board_width,
self.board_height,
model_file=init_model,
use_gpu=self.use_gpu)
#
self.mcts_player = MCTSPlayer(self.policy_value_net.policy_value_fn,
c_puct=self.c_puct,
n_playout=self.n_playout,
is_selfplay=1)
def run():
w = 12
try:
ai_player = AiEngine('pela', 'pbrain-pela.exe', w)
best_policy = PolicyValueNet(
w, w, 'result/pytorch_12_5/current_policy.model')
mcts_player = MCTSPlayer(best_policy.policy_value_fn,
c_puct=5,
n_playout=2000)
game = Game(Board(width=w, height=w, n_in_row=5))
game.start_play(ai_player, mcts_player, start_player=1, is_shown=1)
except KeyboardInterrupt:
print('\n\rquit')
def __init__(self, game_batch_num, model_file=None):
# params of the board and the game
self.size = BOARD_SIZE
use_gpu = False
board = Board(size=self.size, n_in_row=N_IN_ROW)
self.game = Game(board)
# training params
self.learn_rate = 2e-3
self.lr_multiplier = 1.0 # adaptively adjust the learning rate based on KL
self.temp = 1.0 # the temperature param
self.n_playout = 400 # num of simulations for each move
self.c_puct = 5
self.batch_size = 512 # mini-batch size for training
self.data_buffer = deque(maxlen=10000)
self.play_batch_size = 1
self.epochs = 5 # num of train_steps for each update
self.kl_targ = 0.02
self.check_freq = 50
self.game_batch_num = game_batch_num
self.best_win_ratio = 0.0
# num of simulations used for the pure mcts, which is used as
# the opponent to evaluate the trained policy
self.pure_mcts_playout_num = 1000
if model_file:
# start training from an initial policy-value net
self.policy_value_net = PolicyValueNet(size=self.size,
model_file=model_file,
use_gpu=use_gpu)
else:
# start training from a new policy-value net
self.policy_value_net = PolicyValueNet(size=self.size,
use_gpu=use_gpu)
self.mcts_player = MCTSPlayer(self.policy_value_net.policy_value_fn,
c_puct=self.c_puct,
n_playout=self.n_playout,
is_selfplay=1)
def run():
n = N
width, height = SIZE, SIZE
if MCTS_PURE:
player_2 = MCTS_Pure(c_puct=5, n_playout=PLAYOUT)
# print ("Benchmarking the following two models:"+MODEL_1+" Pure MCTS")
elif HUMAN:
player_2 = Human()
# print ("Benchmarking the following two models:"+MODEL_1+" Human")
else:
pass
# print ("Benchmarking the following two models:"+MODEL_1+" "+MODEL_2)
#
# best_policy_2 = PolicyValueNet(width, height, model_file=MODEL_2)
# player_2 = MCTSPlayer(best_policy_2.policy_value_fn,
# c_puct=5,
# n_playout=400) # set larger n_playout for better performance
# player_1=Human()
win_ratios = []
game_batchs = range(50, 1501, 100)
for game_batch in game_batchs:
model = './models/iter_' + str(game_batch) + '.model'
print(model)
policy = PolicyValueNet(width, height, model_file=model)
player_1 = MCTSPlayer(
policy.policy_value_fn, c_puct=5,
n_playout=400) # set larger n_playout for better performance
win_ratio = policy_evaluate(player_1, player_2)
win_ratios.append(win_ratio)
print("The win ratio for " + model + " is: ",
str(100 * win_ratio) + "%")
print(zip(win_ratios, game_batchs))
fig, ax = plt.subplots()
ax.plot(game_batchs, win_ratios)
ax.set(
xlabel='iterations',
ylabel='win ratios',
title='Win ratio of models trained by 5 input states vs. MCTS player')
ax.grid()
fig.savefig("win_ratio.png")
def run():
n = 5
width, height = 8, 8
# model_file = 'best_policy_8_8_5.model'
# model_file = 'best_policy_6_6_4.model'
model_file = 'current_policy.model'
try:
board = Board(width=width, height=height, n_in_row=n)
game = Game(board)
# human player, input your move in the format: 2,3
human = Human()
# ############### human VS AI ###################
# load the trained policy_value_net in either Theano/Lasagne, PyTorch or TensorFlow
# Add FORBIDDEN move player
best_policy = PolicyValueNet(width, height, model_file=model_file)
mcts_player = MCTSPlayer(best_policy.policy_value_fn,
c_puct=5,
n_playout=400)
# load the provided model (trained in Theano/Lasagne) into a MCTS player written in pure numpy
try:
policy_param = pickle.load(open(model_file, 'rb'))
except:
policy_param = pickle.load(open(model_file, 'rb'),
encoding='bytes') # To support python3
# ################ ORIGINAL POLICY and PLAYER ################
# best_policy = PolicyValueNetNumpy(width, height, policy_param)
# mcts_player = MCTSPlayer(best_policy.policy_value_fn,
# c_puct=5,
# n_playout=400) # set larger n_playout for better performance
# uncomment the following line to play with pure MCTS (it's much weaker even with a larger n_playout)
# mcts_player = MCTS_Pure(c_puct=5, n_playout=1000)
mcts_pure = MCTS_Pure(c_puct=5, n_playout=1000)
# set start_player=0 for human first
# game.start_play(human, mcts_player, start_player=1, is_shown=1)
# ############## IMPLEMENTED PURE RL PLAYER ##############
adv_player = QPlayer(board)
# game.start_play(human, adv_player, start_player=1, is_shown=1)
game.start_play(human, adv_player, start_player=1, is_shown=1)
except KeyboardInterrupt:
print('\n\rquit')
def __init__(self,
init_model=None,
board_width=6,
board_height=6,
n_in_row=4,
n_playout=400,
use_gpu=False,
is_shown=False,
output_file_name="",
game_batch_number=1500):
# params of the board and the game
self.board_width = board_width
self.board_height = board_height
self.n_in_row = n_in_row
self.board = Board(width=self.board_width,
height=self.board_height,
n_in_row=self.n_in_row)
self.game = Game(self.board)
# training params
self.learn_rate = 2e-3
self.lr_multiplier = 1.0 # adaptively adjust the learning rate based on KL
self.temp = 1.0 # the temperature param
self.n_playout = n_playout # num of simulations for each move
self.c_puct = 5
self.buffer_size = 10000
self.batch_size = 512 # mini-batch size for training
self.data_buffer = deque(maxlen=self.buffer_size)
self.play_batch_size = 1
self.epochs = 5 # num of train_steps for each update
self.kl_targ = 0.02
self.check_freq = 50
self.game_batch_num = game_batch_number
self.best_win_ratio = 0.0
# num of simulations used for the pure mcts, which is used as
# the opponent to evaluate the trained policy
self.pure_mcts_playout_num = 1000
self.use_gpu = use_gpu
self.is_shown = is_shown
self.output_file_name = output_file_name
self.policy_value_net = PolicyValueNet(self.board_width,
self.board_height,
model_file=init_model,
use_gpu=self.use_gpu)
self.mcts_player = MCTSPlayer(self.policy_value_net.policy_value_fn,
c_puct=self.c_puct,
n_playout=self.n_playout,
is_selfplay=1)
def run():
model_file = 'best_policy_pytorch.model'
try:
board = Board(size=BOARD_SIZE, n_in_row=N_IN_ROW)
game = Game(board)
# ############### human VS AI ###################
best_policy = PolicyValueNet(BOARD_SIZE, model_file)
mcts_player = MCTSPlayer(
best_policy.policy_value_fn, c_puct=5,
n_playout=800) # set larger n_playout for better performance
# human player, input your move in the format: 2,3
human = Human()
# set start_player=0 for human first
game.start_play(human, mcts_player, start_player=1, is_shown=1)
except KeyboardInterrupt:
print('\nQuit')
def run(n_in_row, width, height, model_file, ai_first, n_playout, use_gpu):
try:
board = Board(width=width, height=height, n_in_row=n_in_row) # 产生一个棋盘
game = Game(board) # 加载一个游戏
# ############### 人类 VS AI ###################
best_policy = PolicyValueNet(width,
height,
model_file=model_file,
use_gpu=use_gpu) # 加载最佳策略网络
mcts_player = MCTSPlayer(best_policy.policy_value_fn,
c_puct=5,
n_playout=n_playout) # 生成一个AI玩家
human = Human() # 生成一个人类玩家
# set start_player=0 for human first
game.start_play(human, mcts_player, start_player=ai_first,
is_shown=1) # 开始游戏
except KeyboardInterrupt:
print('\n\rquit')
def run():
n = 6
width, height = 9, 9
model_file = 'best_policy.model' #載入模型
try:
board = Board(width=width, height=height, n_in_row=n)
game = Game(board)
try:
policy_param = pickle.load(open(model_file, 'rb'))
except:
policy_param = pickle.load(open(model_file, 'rb'), encoding = 'bytes') # To support python3
best_policy = PolicyValueNet(width, height, policy_param)
mcts_player = MCTSPlayer(best_policy.policy_value_fn, c_puct=5, n_playout=400) # set larger n_playout for better performance
# human player, input your move in the format: 2,3
human = Human()
# set start_player=0 for human first
game.start_play(human, mcts_player, start_player=1, is_shown=1)
except KeyboardInterrupt:
print('\n\rquit')