def act(self, b, c, turn, temperature=0):
stone_cnt = b[0].sum() + b[1].sum()
b = b.astype(np.bool)
bb = traverse.BitBoard(b)
if 64 - stone_cnt > 12:
x, y, _ = bb.montecarlo(c, 10000, self.strategy)
else:
x, y = traverse.BitBoard(b.astype(np.bool)).traverse(
c, self.strategy)
return x * 8 + y
def main():
board, prob = T.BitBoard(B.init()).get_score_prob2(Color.Black, 54, 1000)
print(B.stringify(board.to_board()))
f = plt.figure()
a = f.add_subplot(111)
a.plot(np.arange(127) - 64, np.array(prob))
a.set_ylim((0, 1))
a.set_xlim(-64, 64)
a.set_xlabel('#BLACK - #WHITE')
a.set_ylabel('probability')
f.savefig("/Users/kikura/Desktop/prob.png")
def main():
parser = argparse.ArgumentParser(description='')
parser.add_argument('--init', '-i', help='path to initial olayer model')
parser.add_argument('--opponent')
parser.add_argument('--gpu',
'-g',
type=int,
default=-1,
help='GPU device ID')
parser.add_argument('--epoch',
'-e',
type=int,
default=10000,
help='# of epoch')
parser.add_argument('--batch_size',
type=int,
default=256,
help='size of mini-batch')
parser.add_argument('--adam_eps',
type=float,
default=1e-2,
help='parameter eps in adam')
parser.add_argument('--adam_alpha',
type=float,
default=1e-4,
help='parameter alpha in adam')
parser.add_argument('--density',
type=int,
default=1,
help='density of cnn kernel')
parser.add_argument('--no_bn',
dest='use_bn',
action='store_false',
default=True)
parser.add_argument('--draw',
dest='draw',
action='store_true',
default=False)
parser.add_argument('--out', default='')
parser.set_defaults(test=False)
args = parser.parse_args()
batch_size = args.batch_size
# log directory
out = datetime.datetime.now().strftime('%m%d')
if args.out:
out = out + '_' + args.out
out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs_rl", out))
os.makedirs(os.path.join(out_dir, 'models'), exist_ok=True)
player_model = RLPolicy(use_bn=args.use_bn)
# opponent_model = RLPolicy(use_bn=args.use_bn)
opponent_model = RolloutPolicy()
# load player model
serializers.load_hdf5(args.init, player_model)
# gpu
if args.gpu >= 0:
cuda.get_device(args.gpu).use()
player_model.to_gpu()
opponent_model.to_gpu()
# setting
with open(os.path.join(out_dir, 'setting.txt'), 'w') as f:
for k, v in args._get_kwargs():
print('{} = {}'.format(k, v))
f.write('{} = {}\n'.format(k, v))
# optimizer
optimizer = chainer.optimizers.Adam(alpha=args.adam_alpha,
eps=args.adam_eps)
optimizer.setup(player_model)
# start training
start = time.time()
# opponent model
if args.opponent is None:
opponent = args.init
else:
opponent = args.opponent
opponent_models = [opponent]
win_rate_summary = []
for epoch in range(args.epoch):
# load opponent model
if epoch % 1000 == 0:
serializers.load_hdf5(np.random.choice(opponent_models),
opponent_model)
# initialize
states = np.zeros(shape=(batch_size, 80, 6, 8, 8))
plies = np.zeros(shape=(batch_size, 80), dtype='int64')
ply_nums = np.zeros(shape=batch_size, dtype='int64')
results = np.zeros(shape=batch_size)
# simulation (self-play)
for player_color in [Color.Black, Color.White]:
models = {
player_color: player_model,
1 - player_color: opponent_model
}
x_batch = [
board.to_state(board.init(), 0, 0)
for _ in range(batch_size // 2)
]
turn = 0
c = Color.Black
pass_cnts = np.zeros(shape=batch_size // 2)
while True:
if min(pass_cnts) >= 2:
break
if c == player_color:
scores = models[c].predict(
models[c].xp.array(x_batch, 'float32'), False)
scores = cuda.to_cpu(scores.data)
for i in range(batch_size // 2):
# gameが終わったか判定
if pass_cnts[i] == 2:
continue
b = x_batch[i][:2]
valid_mask = x_batch[i][2].ravel()
if valid_mask.sum() == 0:
plies[(batch_size // 2) * player_color + i, turn] = -1
pass_cnts[i] += 1
x_batch[i] = board.to_state(b, 1 - c, turn + 1)
else:
stone_cnt = b[0:2].sum()
if c == player_color:
if stone_cnt >= 64 - 8:
# 残り12手は探索で。
# print('in zentansaku', stone_cnt)
if args.draw:
x, y = traverse.BitBoard(b.astype(
np.bool)).traverse(c, 3)
else:
x, y = traverse.BitBoard(b.astype(
np.bool)).traverse(c, 1)
ply = x * 8 + y
else:
pred = softmax(scores[i].astype(np.float64),
mask=valid_mask,
T=1)
ply = np.random.choice(64, p=pred)
states[(batch_size // 2) * player_color + i,
turn, :, :, :] = x_batch[i]
plies[(batch_size // 2) * player_color + i,
turn] = ply
ply_nums[(batch_size // 2) * player_color +
i] += 1
x = ply // 8
y = ply % 8
else:
stone_cnt = b[0].sum() + b[1].sum()
b = b.astype(np.bool)
bb = traverse.BitBoard(b)
if 64 - stone_cnt > 12:
x, y = bb.montecarlo(c, 10000, 1)
else:
x, y = traverse.BitBoard(b.astype(
np.bool)).traverse(c, 1)
if not board.is_valid(b, c, x, y):
print(valid_mask)
print(scores[i])
print(softmax(scores[i], mask=valid_mask, T=1))
raise ValueError('invalid ply')
x_batch[i] = board.to_state(board.put(b, c, x, y),
1 - c, turn + 1)
pass_cnts[i] = 0
c = 1 - c
turn += 1
# check win/lose
for i, b in enumerate(x_batch):
num_black = b[0].sum()
num_white = b[1].sum()
if args.draw:
diff = abs(num_black - num_white)
if diff <= 3:
res = 1
elif diff <= 10:
res = 0
else:
res = -0.5
results[(batch_size // 2) * player_color + i] = res
else:
res = np.sign(num_black - num_white)
res = res if player_color == Color.Black else -res
results[(batch_size // 2) * player_color +
i] = res if res >= 0 else res / 2
# train (policy gradient)
optimizer.update(player_model, states, plies, results, ply_nums)
if args.draw:
win_rate = np.mean(np.array(results) > 0)
else:
win_rate = np.mean(np.array(results) >= 0)
progress_report(start, epoch, batch_size, win_rate)
win_rate_summary.append(win_rate)
if epoch % 100 == 0:
serializers.save_hdf5(
os.path.join(out_dir, "models",
"rl_policy_{}.model".format(epoch)), player_model)
print('\nwin_rate_summary {}'.format(np.mean(win_rate_summary)))
win_rate_summary = []
from reversi import board as B, traverse as T, Color
import numpy as np
bitboard = T.BitBoard(B.init())
for max_num_stone in range(54, 4, -1):
bitboards, probs = bitboard.get_score_prob2(Color.Black, max_num_stone,
100)
np.savez_compressed("./dataset.npz",
boards=np.array(bitboards),
probs=np.array(probs))
print("save >> {0}".format(max_num_stone))
def find_ply_montecarlo_traverse_win(board: B, color: C):
if board[:2].sum() > 52:
return T.BitBoard(board).traverse(color, 1)
else:
return T.BitBoard(board).montecarlo(color, 1000, 1)
def find_ply_random(board: B, color: C):
hands = T.BitBoard(board).find_next(color)
return (-1, -1) if len(hands) == 0 else hands[random.randint(
0,
len(hands) - 1)]
def find_ply_montecarlo_draw(board: B, color: C):
return T.BitBoard(board).montecarlo(color, 1000, 3)
