def load(self, path):
pgn = open(path)
offsets = []
for offset, headers in chess.pgn.scan_headers(pgn):
offsets.append(offset)
for offset in offsets:
pgn.seek(offset)
game = chess.pgn.read_game(pgn)
exporter = chess.pgn.StringExporter()
pgn_string = game.accept(exporter)
game_headers = self._get_headers_columns(game.headers)
game_id = [x for x in (Game.insert(pgn=pgn_string, **game_headers).returning(Game.id)).execute()][0][0]
node = game
while not node.is_end():
next_node = node.variations[0]
fen = node.board().fen()
hash_key = hashlib.md5(fen.encode()).hexdigest()
position_result = (Position.select().where(Position.hash == hash_key).execute())
if not position_result:
position_id = [x for x in (Position.insert(hash=hash_key).returning(Position.id)).execute()][0][0]
else:
position_id = position_result[0].id
PositionGame.insert(position_id=position_id, game_id=game_id).execute()
node = next_node
pgn.close()
def get_games_from_file(self, filename):
pgn = open(filename, errors='ignore')
games = []
for offset in chess.pgn.scan_offsets(pgn):
pgn.seek(offset)
games.append(chess.pgn.read_game(pgn))
return games
def load_generic_text(directory):
'''Generator that yields text raw from the directory.'''
files = find_files(directory)
for filename in files:
text = ""
k = 0
pgn = open(filename)
for offset, headers in chess.pgn.scan_headers(pgn):
pgn.seek(offset)
game = chess.pgn.read_game(pgn)
node = game
nm = 1
while not node.is_end():
board_fen = replace_tags(node.board().fen())
next_node = node.variation(0)
label_san = node.board().san(next_node.move)
text += board_fen + ":" + label_san + "\n"
nm += 1
node = next_node
if k % 1000 == 0 and k > 1:
print("Saving file: " + filename + "-" + str(k) + ".txt")
y = []
for index, item in enumerate(text):
y.append(text[index])
y = np.array(y)
np.savetxt(filename + "-" + str(k) + ".txt",
y.reshape(1, y.shape[0]),
delimiter="",
newline="\n",
fmt="%s")
text = ""
k += 1
pgn.close()
yield filename
def load_generic_text(directory):
'''Generator that yields text raw from the directory.'''
files = find_files(directory)
text = " "
for filename in files:
k = 0
pgn = open(filename)
for offset, headers in chess.pgn.scan_headers(pgn):
pgn.seek(offset)
game = chess.pgn.read_game(pgn)
node = game
while not node.is_end():
next_node = node.variation(0)
label_san = node.board().san(next_node.move)
if " " + label_san + " " not in text:
text += label_san + " "
node = next_node
if k % 100 == 0 and k > 1:
print ("Labeling file: " + filename + ", step: " + str(k))
k += 1
pgn.close()
y = []
for index, item in enumerate(text):
y.append(text[index])
y = np.array(y)
np.savetxt("labels.txt", y.reshape(1, y.shape[0]), delimiter="", newline="\n", fmt="%s")
def parser(file, elo=400, draws=False, time_control=True):
pgn = open(file)
games = []
results = []
offsets = []
counter = 0
while True:
offset = pgn.tell()
headers = chess.pgn.read_headers(pgn)
if headers is None:
break
if headers.get("Result") not in ["1-0", "0-1", "1/2-1/2"]:
continue
if not draws:
if headers.get("Result") == "1/2-1/2":
continue
elo_white = headers.get("WhiteElo")
elo_black = headers.get("BlackElo")
if elo_white is None or not elo_white.isnumeric():
elo_white = 0
if elo_black is None or not elo_black.isnumeric():
elo_black = 0
time_control = headers.get("TimeControl")
if time_control:
if not time_control[0:4].isnumeric():
if not time_control[0:3].isnumeric():
continue
if int(time_control[0:3]) < 180:
continue
if int(elo_black) >= elo or int(elo_white) >= elo:
offsets.append(offset)
result = headers.get("Result")
if result == "1-0":
results.append("white")
elif result == "0-1":
results.append("black")
else:
print(result)
results.append("draw")
counter += 1
if counter > 50000:
break
for offset in offsets:
pgn.seek(offset)
games.append(chess.pgn.read_game(pgn))
return games, results
def test_scan_headers(self):
with open("data/games/kasparov-deep-blue-1997.pgn") as pgn:
offsets = (offset for offset, headers in chess.pgn.scan_headers(pgn)
if headers["Result"] == "1/2-1/2")
first_drawn_game_offset = next(offsets)
pgn.seek(first_drawn_game_offset)
first_drawn_game = chess.pgn.read_game(pgn)
self.assertEqual(first_drawn_game.headers["Site"], "03")
self.assertEqual(first_drawn_game.variation(0).move, chess.Move.from_uci("d2d3"))
def get_games_from_file(filename):
pgn = open(filename, errors='ignore')
offsets = list(chess.pgn.scan_offsets(pgn))
n = len(offsets)
print(f"found {n} games")
games = []
for offset in offsets:
pgn.seek(offset)
games.append(chess.pgn.read_game(pgn))
return games
def load_game(self, index):
if index >= 0 and index < len(self.entries):
entry = self.entries[index]
with open(self.filename) as pgn:
offset = entry.pgn_offset
pgn.seek(offset)
game = chess.pgn.read_game(pgn)
self.index_current_game = index
return game
else:
raise ValueError("no game for supplied index in database")
def load_generic_text(directory, type_file):
'''Generator that yields text raw from the directory.'''
files = find_files(directory)
print(files)
for filename in files:
print("Parsing file: " + filename)
board_text = ""
movement_text = ""
k = 0
pgn = open(filename)
for offset, headers in chess.pgn.scan_headers(pgn):
pgn.seek(offset)
game = chess.pgn.read_game(pgn)
node = game
nm = 1
while not node.is_end():
board_fen = replace_tags(node.board().fen())
next_node = node.variation(0)
label_san = node.board().san(next_node.move)
board_text += " ".join(board_fen) + "\n"
movement_text += " ".join(label_san) + "\n"
nm += 1
node = next_node
if k % 1000 == 0 and k > 1:
print("Saving: " + filename + " " + str(k))
print("Saving: " + filename + " " + str(k))
y = []
m = []
for index, item in enumerate(board_text):
y.append(board_text[index])
for index, item in enumerate(movement_text):
m.append(movement_text[index])
y = np.array(y)
m = np.array(m)
with open(
"nmt/datasets/" + type_file + "/" + type_file + ".gm",
'a') as f_handle:
np.savetxt(f_handle,
y.reshape(1, y.shape[0]),
delimiter="",
newline="\n",
fmt="%s")
with open(
"nmt/datasets/" + type_file + "/" + type_file + ".mv",
'a') as f_handle:
np.savetxt(f_handle,
m.reshape(1, m.shape[0]),
delimiter="",
newline="\n",
fmt="%s")
board_text = ""
movement_text = ""
k += 1
pgn.close()
def offsets_to_df(offsets, pgn):
print('Parsing {} games.'.format(len(offsets)))
rows = []
with click.progressbar(offsets, label="Parsing pgn") as offsets:
for offset in offsets:
pgn.seek(offset)
game = chess.pgn.read_game(pgn)
rows.extend(game_to_rows(game))
return pd.DataFrame(rows)
def read_file(self, filename):
pgn = open(filename, errors='ignore')
futures = deque()
for offset in chess.pgn.scan_offsets(pgn):
pgn.seek(offset)
game = chess.pgn.read_game(pgn)
futures.append(self.executor.submit(get_buffer, game, self.config))
print(f"found {len(futures)} games")
while futures:
yield futures.popleft() # no more memleak
def load_game(self, index):
print(self.entries)
if index >= 0 and index < len(self.entries):
entry = self.entries[index]
with open(self.filename) as pgn:
offset = entry.pgn_offset
pgn.seek(offset)
game = chess.pgn.read_game(pgn)
self.index_current_game = index
return game
else:
raise ValueError("no game for supplied index in database")
def test_scan_offsets(self):
with open("data/games/kasparov-deep-blue-1997.pgn") as pgn:
offsets = list(chess.pgn.scan_offsets(pgn))
self.assertEqual(len(offsets), 6)
pgn.seek(offsets[0])
first_game = chess.pgn.read_game(pgn)
self.assertEqual(first_game.headers["Event"], "IBM Man-Machine, New York USA")
self.assertEqual(first_game.headers["Site"], "01")
pgn.seek(offsets[5])
sixth_game = chess.pgn.read_game(pgn)
self.assertEqual(sixth_game.headers["Event"], "IBM Man-Machine, New York USA")
self.assertEqual(sixth_game.headers["Site"], "06")
def get_games_from_file(filename: str) -> List[chess.pgn.Game]:
"""
:param str filename: file containing the pgn game data
:return list(pgn.Game): chess games in that file
"""
pgn = open(filename, errors='ignore')
offsets = list(chess.pgn.scan_offsets(pgn))
n = len(offsets)
print(f"found {n} games")
games = []
for offset in offsets:
pgn.seek(offset)
games.append(chess.pgn.read_game(pgn))
return games
def update_game(self, idx, game_tree):
old_length = self.delete_game_for_update_at(idx)
dos_newlines = False
if "\r\n".encode() in open(self.filename,"rb").read():
dos_newlines = True
if(dos_newlines):
game_str = (str(game_tree.root())).replace('\n','\r\n')
game_str = (game_str + '\r\n\r\n').encode('utf-8')
else:
game_str = (str(game_tree.root())+"\n\n").encode('utf-8')
length = len(game_str)
offset = self.entries[idx].pgn_offset
# we can't mmap an empty file
# the file is however empty, if the current
# game was the only one in the database
# just append it
current_filesize = os.stat(self.filename).st_size
print("current fs: "+str(current_filesize))
if current_filesize == 0:
print("file is empty")
self.entries = []
self.append_game(game_tree)
else:
pgn = open(self.filename, 'r+')
m=mm.mmap(pgn.fileno(),0)
size = len(m)
new_size = size + length
m.flush()
m.close()
pgn.seek(size)
pgn.write('A'*length)
pgn.flush()
m=mm.mmap(pgn.fileno(),0)
m.move(offset+length,offset,size-offset)
m.seek(offset)
m.write(game_str)
m.flush()
m.close()
pgn.close()
for i in range(idx+1, len(self.entries)):
self.entries[i].pgn_offset += (length-old_length)
self.checksum = crc32_from_file(self.filename)
self.entries[idx] = Entry(offset, game_tree.root().headers)
def get_games_from_file(filename):
"""
:param str filename: file containing the pgn game data
:return list(pgn.Game): chess games in that file
"""
pgn = open(filename, errors='ignore')
offsets = list(chess.pgn.scan_offsets(pgn))
n = len(offsets)
print(f"found {n} games")
games = []
for offset in offsets:
pgn.seek(offset)
games.append(chess.pgn.read_game(pgn))
return games
def update_game(self, idx, game_tree):
old_length = self.delete_game_for_update_at(idx)
dos_newlines = False
if "\r\n".encode() in open(self.filename, "rb").read():
dos_newlines = True
if (dos_newlines):
game_str = (str(game_tree.root())).replace('\n', '\r\n')
game_str = (game_str + '\r\n\r\n').encode('utf-8')
else:
game_str = (str(game_tree.root()) + "\n\n").encode('utf-8')
length = len(game_str)
offset = self.entries[idx].pgn_offset
# we can't mmap an empty file
# the file is however empty, if the current
# game was the only one in the database
# just append it
current_filesize = os.stat(self.filename).st_size
if current_filesize == 0:
self.entries = []
self.append_game(game_tree)
else:
pgn = open(self.filename, 'r+')
m = mm.mmap(pgn.fileno(), 0)
size = len(m)
new_size = size + length
m.flush()
m.close()
pgn.seek(size)
pgn.write('A' * length)
pgn.flush()
m = mm.mmap(pgn.fileno(), 0)
m.move(offset + length, offset, size - offset)
m.seek(offset)
m.write(game_str)
m.flush()
m.close()
pgn.close()
for i in range(idx + 1, len(self.entries)):
self.entries[i].pgn_offset += (length - old_length)
self.checksum = crc32_from_file(self.filename)
self.entries[idx] = Entry(offset, game_tree.root().headers)
def filter_pgn_file_for_games(
pgn: TextIO,
filters: Dict[str, List[Any]] = {},
verbose_period: Optional[int] = None,
) -> Generator[chess.pgn.Game, None, None]:
count = 0
while True:
offset = pgn.tell()
headers = chess.pgn.read_headers(pgn)
if headers is None:
break
for key, vals in filters.items():
if headers.get(key) not in vals:
break
else:
pgn.seek(offset)
game = chess.pgn.read_game(pgn)
count += 1
if verbose_period is not None and count % verbose_period == 0:
print(f"{count} games processed")
yield game
def get_games_from_file(filename):
"""
:param str filename: file containing the pgn game data
:return list(pgn.Game): chess games in that file
"""
pgn = open(filename, errors='ignore')
offsets = []
while True:
offset = pgn.tell()
headers = chess.pgn.read_headers(pgn)
if headers is None:
break
offsets.append(offset)
n = len(offsets)
print(f"found {n} games")
games = []
for offset in offsets:
pgn.seek(offset)
games.append(chess.pgn.read_game(pgn))
return games
def gen(pgn_file_path):
with open(pgn_file_path) as pgn:
# Start from random position in the file
pgn.seek(0, 2)
pgn.seek(random.randint(0, pgn.tell()))
chess.pgn.read_headers(pgn)
while True:
game = chess.pgn.read_game(pgn)
if not game:
pgn.seek(0)
continue
'''
result_header = game.headers['Result']
game_value_for_white = 0
if result_header == '*':
continue
elif result_header == '1-0':
game_value_for_white = 1
elif result_header == '0-1':
game_value_for_white = -1
else:
game_value_for_white = 0
'''
board = game.board()
for node in game.mainline():
board.push(node.move)
eval = node.eval()
if not eval:
break
eval = eval.pov(not board.turn).score()
if not eval:
continue
try:
X = get_halfkp_indeces(board)
except:
print(f'Would have crashed: {board}')
continue
# y = game_value_for_white if board.turn == chess.WHITE else -game_value_for_white
# y = eval if board.turn == chess.WHITE else -eval
yield (X[0], X[1]), eval / 100
def read(filename, data):
pgn = open(filename)
cnt = 0
while True:
cnt += 1
if cnt % 10000 == 0:
print(cnt)
offset = pgn.tell()
headers = chess.pgn.read_headers(pgn)
if headers is None:
break
point = 0
if headers['Result'] == '1-0':
point = 1
if headers['Result'] == '1/2-1/2':
point = 0.5
date = headers['UTCDate']
if date not in data:
data[date] = {}
data[date]['games'] = [1, 0]
data[date]['traps'] = [[0, 0], [0, 0], [0, 0], [0, 0], [0, 0],
[0, 0]]
"""
0: Oh no my queen, Stafford
1: Oh no my knight, Stafford
2: Stafford: 5.e5 Ne4 6.d4 Qh4 7.g3 Ng3
3: Punishing natural development, Stafford
4: Punishing capture hxg4, Stafford
5: Englund gambit trap
"""
else:
data[date]['games'][0] += 1
data[date]['games'][1] += point
opening = headers["Opening"]
if opening not in data[date]:
data[date][opening] = [1, 0]
else:
data[date][opening][0] += 1
data[date][opening][1] += point
if opening == 'Englund Gambit' or opening == 'Russian Game: Stafford Gambit':
pgn.seek(offset)
mygame = chess.pgn.read_game(pgn)
for i in range(10):
if mygame.next() != None:
mygame = mygame.next()
if mygame.board().fen(
) == 'rnbqk2r/ppp1Pppp/8/8/8/5N2/PPP1PbPP/RNBQKB1R w KQkq - 0 6':
data[date]['traps'][5][0] += 1
data[date]['traps'][5][1] += point
print(date, data[date]['traps'])
mygame = mygame.game()
for i in range(13):
if mygame.next() != None:
mygame = mygame.next()
if mygame.board().fen(
) == 'r1bBk2r/ppp2ppp/2p5/2b5/4n3/3P4/PPP2PPP/RN1QKB1R b KQkq - 0 7':
data[date]['traps'][0][0] += 1
data[date]['traps'][0][1] += point
print(date, data[date]['traps'])
mygame = mygame.game()
for i in range(12):
if mygame.next() != None:
mygame = mygame.next()
if mygame.board().fen(
) == 'r1bqk2r/ppp2ppp/2p5/2b1P3/4n3/3P4/PPP2PPP/RNBQKB1R w KQkq - 1 7':
data[date]['traps'][1][0] += 1
data[date]['traps'][1][1] += point
print(date, data[date]['traps'])
mygame = mygame.game()
for i in range(14):
if mygame.next() != None:
mygame = mygame.next()
if mygame.board().fen(
) == 'r1b1kb1r/ppp2ppp/2p5/4P3/3P3q/6n1/PPP2P1P/RNBQKB1R w KQkq - 0 8':
data[date]['traps'][2][0] += 1
data[date]['traps'][2][1] += point
print(date, data[date]['traps'])
mygame = mygame.game()
for i in range(16):
if mygame.next() != None:
mygame = mygame.next()
if mygame.board().fen(
) == 'r1b1k2r/ppp2ppp/2p5/2b5/2B1P2q/2N4P/PPPP1nP1/R1BQ1RK1 w kq - 0 9':
data[date]['traps'][3][0] += 1
data[date]['traps'][3][1] += point
print(date, data[date]['traps'])
mygame = mygame.game()
for i in range(17):
if mygame.next() != None:
mygame = mygame.next()
if mygame.board().fen() == 'r1b1k2r/ppp2pp1/2p5/2b4p/3qP1P1/2N5/PPPPBPP1/R1BQ1RK1 b kq - 0 9' or \
mygame.board().fen() == 'r1b1k2r/ppp2pp1/2pq4/2b4p/4P1P1/3P4/PPP1BPP1/RNBQ1RK1 b kq - 0 9':
data[date]['traps'][4][0] += 1
data[date]['traps'][4][1] += point
print(date, data[date]['traps'])
return data
def get_end_offset(self):
with open(self.filename, 'r') as pgn:
pgn.seek(0, os.SEEK_END)
end_offset = pgn.tell()
return end_offset
if __name__ == '__main__':
pathPgn = './output.pgn'
if sys.argv[1:]:
pathPgn = sys.argv[1]
print "opening PGN: %s" % pathPgn
pgn = open(pathPgn)
lineCur = 1
lineToOffset = {}
while 1:
lineToOffset[lineCur] = pgn.tell()
if not pgn.readline():
break;
lineCur += 1
pgn.seek(0)
print lineToOffset
sys.exit(-1)
gameNum = 1
while 1:
game = chess.pgn.read_game(pgn)
fen = game.headers["FEN"]
print "offset:%d, game:%d, fen:%s" % (pgn.tell(), gameNum, fen)
m = re.match(regexFen, fen)
if not m:
print "%s is not valid" % fen
fix(filename)
with open("../../AepliBase.pgn") as pgn:
offsets_gen = chess.pgn.scan_offsets(pgn)
offsets = []
try:
while len(offsets) < N:
# skip some
for i in range(200):
offset = next(offsets_gen)
offsets.append(offset)
sys.stdout.write('.')
sys.stdout.flush()
except:
pass
print(len(offsets), ' offsets found')
for offset in offsets:
pgn.seek(offset)
g = chess.pgn.read_game(pgn)
games.append(g)
sys.stdout.write('.')
sys.stdout.flush()
print(len(games), ' games parsed')
with open('qanim.js', 'w') as out:
out.write('moves=[')
for iMoveWanted in range(depth):
print('\n', iMoveWanted)
piece_move_counts = [[[[[0] * S for y1 in range(S)] for x2 in range(S)]
for y2 in range(S)] for p in range(P)]
for g in games:
sys.stdout.write('.')
sys.stdout.flush()
moves = list(g.main_line())
def read_game(self, idx):
self.env.reset()
self.black = ChessPlayer(self.config, self.model)
self.white = ChessPlayer(self.config, self.model)
files = find_pgn_files(self.config.resource.play_data_dir)
if len(files) > 0:
random.shuffle(files)
filename = files[0]
pgn = open(filename, errors='ignore')
size = os.path.getsize(filename)
pos = random.randint(0, size)
pgn.seek(pos)
line = pgn.readline()
offset = 0
# Parse game headers.
while line:
if line.isspace() or line.startswith("%"):
line = pgn.readline()
continue
# Read header tags.
tag_match = TAG_REGEX.match(line)
if tag_match:
offset = pgn.tell()
break
line = pgn.readline()
pgn.seek(offset)
game = chess.pgn.read_game(pgn)
node = game
result = game.headers["Result"]
actions = []
while not node.is_end():
next_node = node.variation(0)
actions.append(node.board().uci(next_node.move))
node = next_node
pgn.close()
k = 0
observation = self.env.observation
while not self.env.done and k < len(actions):
if self.env.board.turn == chess.BLACK:
action = self.black.sl_action(observation, actions[k])
else:
action = self.white.sl_action(observation, actions[k])
board, info = self.env.step(action)
observation = board.fen()
k += 1
self.env.done = True
if not self.env.board.is_game_over() and result != '1/2-1/2':
self.env.resigned = True
if result == '1-0':
self.env.winner = Winner.white
elif result == '0-1':
self.env.winner = Winner.black
else:
self.env.winner = Winner.draw
self.finish_game()
self.save_play_data(write=idx % self.config.play_data.nb_game_in_file == 0)
self.remove_play_data()
return self.env
if __name__ == '__main__':
pathPgn = './output.pgn'
if sys.argv[1:]:
pathPgn = sys.argv[1]
print "opening PGN: %s" % pathPgn
pgn = open(pathPgn)
lineCur = 1
lineToOffset = {}
while 1:
lineToOffset[lineCur] = pgn.tell()
if not pgn.readline():
break
lineCur += 1
pgn.seek(0)
print lineToOffset
sys.exit(-1)
gameNum = 1
while 1:
game = chess.pgn.read_game(pgn)
fen = game.headers["FEN"]
print "offset:%d, game:%d, fen:%s" % (pgn.tell(), gameNum, fen)
m = re.match(regexFen, fen)
if not m:
print "%s is not valid" % fen
fix(filename)
def get_end_offset(self):
with open(self.filename,'r') as pgn:
pgn.seek(0,os.SEEK_END)
end_offset = pgn.tell()
print("determined end offset: "+str(end_offset))
return end_offset
def state_action_sl(self, loop=True, featurized=False, board="both"):
"""
Returns (state, action) tuple from white's perspective - flips black's perspective to match
- state: np.array [12 pieces x 64 squares]
- piece order: wp wn wb wr wq wk bp bn bb br bq bk
- square order: a1 b1 c1 ... h8
- action: [np.array [1 x 64 squares], np.array [1 x 64 squares]] representing [from_board, to_board]
"""
from_board = False
to_board = False
if board == "both":
from_board = True
to_board = True
elif board == "from":
from_board = True
elif board == "to":
to_board = True
elif board == "full":
pass
idx_batch = 0
with open(self.filename) as pgn:
S = []
A_from = []
A_to = []
A_combined = []
while True:
game = chess.pgn.read_game(pgn)
if game is None:
if not loop:
break
print("\n******************************************")
print("********** LOOPING OVER DATASET **********")
print("******************************************\n")
pgn.seek(0)
continue
num_moves = int(game.headers["PlyCount"])
board = game.board()
node = game.root()
if num_moves <= 4:
continue
# Make sure game was played all the way through
last_node = game.root()
while last_node.variations:
last_node = last_node.variations[0]
if "forfeit" in last_node.comment:
continue
black_turn = False
while node.variations:
# Play white
s = state_from_board(board,
featurized=featurized,
black=black_turn)
move = node.variations[0].move
a_from, a_to = action_from_move(move, black=black_turn)
board.push(move)
black_turn = not black_turn
node = node.variations[0]
S.append(s)
A_from.append(a_from)
A_to.append(a_to)
if not (from_board or to_board):
a_combined = np.outer(a_from, a_to).flatten()
A_combined.append(a_combined)
idx_batch += 1
if idx_batch >= BATCH_SIZE and len(S) >= POOL_SIZE:
# Shuffle moves in game
idx = list(np.random.permutation(len(S)))
S_shuffle = [S[i] for i in idx]
A_from_shuffle = [A_from[i] for i in idx]
A_to_shuffle = [A_to[i] for i in idx]
S = S_shuffle[BATCH_SIZE:]
A_from = A_from_shuffle[BATCH_SIZE:]
A_to = A_to_shuffle[BATCH_SIZE:]
if not not A_combined:
A_combined_shuffle = [A_combined[i] for i in idx]
A_combined = A_combined_shuffle[BATCH_SIZE:]
idx_batch = 0
if from_board and to_board:
yield (np.array(S_shuffle[:BATCH_SIZE]), \
[np.array(A_from_shuffle[:BATCH_SIZE]), \
np.array(A_to_shuffle[:BATCH_SIZE])])
elif from_board:
yield (np.array(S_shuffle[:BATCH_SIZE]), \
np.array(A_from_shuffle[:BATCH_SIZE]))
elif to_board:
yield ([np.array(S_shuffle[:BATCH_SIZE]), \
np.array(A_from_shuffle[:BATCH_SIZE]).reshape(BATCH_SIZE,1,NUM_ROWS,NUM_COLS)], \
np.array(A_to_shuffle[:BATCH_SIZE]))
else:
yield (np.array(S_shuffle[:BATCH_SIZE]), \
np.array(A_combined_shuffle[:BATCH_SIZE]))
def black_state_action_sl(self, loop=True, featurized=False):
"""
Returns (state, action) tuple from black's perspective
- state: np.array [12 pieces x 64 squares]
- piece order: wp wn wb wr wq wk bp bn bb br bq bk
- square order: a1 b1 c1 ... h8
- action: [np.array [1 x 64 squares], np.array [1 x 64 squares]] representing [from_board, to_board]
"""
BATCH_SIZE = 32
idx_batch = 0
with open(self.filename) as pgn:
S = []
A_from = []
A_to = []
while True:
game = chess.pgn.read_game(pgn)
if game is None:
if not loop:
break
print("\n******************************************")
print("********** LOOPING OVER DATASET **********")
print("******************************************\n")
pgn.seek(0)
continue
num_moves = int(game.headers["PlyCount"])
board = game.board()
node = game.root()
if num_moves <= 5:
continue
# Make sure game was played all the way through
last_node = game.root()
while last_node.variations:
last_node = last_node.variations[0]
if "forfeit" in last_node.comment:
continue
# Play white
move = node.variations[0].move
board.push(move)
node = node.variations[0]
while node.variations:
s = state_from_board(board, featurized=featurized)
move = node.variations[0].move
a_from, a_to = action_from_move(move)
# Play black
board.push(move)
# Play white
node = node.variations[0]
if node.variations:
move = node.variations[0].move
board.push(move)
if node.variations:
node = node.variations[0]
S.append(s)
A_from.append(a_from)
A_to.append(a_to)
idx_batch += 1
if idx_batch == BATCH_SIZE:
# Shuffle moves in game
idx = list(np.random.permutation(len(S)))
S_shuffle = [S[i] for i in idx]
A_from_shuffle = [A_from[i] for i in idx]
A_to_shuffle = [A_to[i] for i in idx]
S = []
A_from = []
A_to = []
idx_batch = 0
yield np.array(S_shuffle), [
np.array(A_from_shuffle),
np.array(A_to_shuffle)
]
def state_value(self, loop=True, featurized=False, board='both'):
"""
Returns (state, action) tuple from white's perspective - flips black's perspective to match
- state: np.array [12 pieces x 64 squares]
- piece order: wp wn wb wr wq wk bp bn bb br bq bk
- square order: a1 b1 c1 ... h8
- action: [np.array [1 x 64 squares], np.array [1 x 64 squares]] representing [from_board, to_board]
"""
idx_batch = 0
with open(self.filename) as pgn:
S = []
R = []
while True:
game = chess.pgn.read_game(pgn)
if game is None:
if not loop:
break
print("\n******************************************")
print("********** LOOPING OVER DATASET **********")
print("******************************************\n")
pgn.seek(0)
continue
num_moves = int(game.headers["PlyCount"])
board = game.board()
node = game.root()
if num_moves <= 4:
continue
# Make sure game was played all the way through
last_node = game.root()
while last_node.variations:
last_node = last_node.variations[0]
if "forfeit" in last_node.comment:
continue
z = 0
white_score = game.headers["Result"].split("-")[0].split("/")
if len(white_score) == 1:
z = 2 * int(white_score[0]) - 1
black_turn = False
while node.variations:
# Play white
s = state_from_board(board,
featurized=featurized,
black=black_turn)
move = node.variations[0].move
r = z if not black_turn else -z
board.push(move)
black_turn = not black_turn
node = node.variations[0]
S.append(s)
R.append(r)
idx_batch += 1
if idx_batch >= BATCH_SIZE and len(S) >= POOL_SIZE:
# Shuffle moves in game
idx = list(np.random.permutation(len(S)))
S_shuffle = [S[i] for i in idx]
R_shuffle = [R[i] for i in idx]
S = S_shuffle[BATCH_SIZE:]
R = R_shuffle[BATCH_SIZE:]
idx_batch = 0
yield np.array(S_shuffle[:BATCH_SIZE]), np.array(
R_shuffle[:BATCH_SIZE])
def random_black_state(self):
"""
Returns (state, reward) tuple at black's turn from white's perspective
- state: np.array [12 pieces x 8 rows x 8 cols]
- piece order: wp wn wb wr wq wk bp bn bb br bq bk
- row order: a b c ... h
- col order: 1 2 3 ... 8
- reward: GAMMA^moves_remaining * {-1, 0, 1} (lose, draw, win)
"""
with open(self.filename) as pgn:
while True:
game = chess.pgn.read_game(pgn)
if game is None:
if not self.loop:
break
pgn.seek(0)
self.num_games = self.idx_game
self.idx_game = 0
continue
num_moves = int(game.headers["PlyCount"])
if num_moves < 2:
continue
# Make sure game was played all the way through
last_node = game.root()
while last_node.variations:
last_node = last_node.variations[0]
if "forfeit" in last_node.comment:
continue
# Choose a random black-turn state
# if self.test_set:
# if self.num_games:
# idx_move = self.idx_moves[self.idx_game]
# else:
# idx_move = random.randint(1, num_moves // 2) * 2 - 1
# self.idx_moves.append(idx_move)
# else:
# idx_move = random.randint(1, num_moves // 2) * 2 - 1
idx_move = random.randint(1, num_moves // 2) * 2 - 1
moves_remaining = (num_moves - idx_move) // 2
# Play moves up to idx_move
board = game.board()
node = game.root()
for i in range(idx_move):
board.push(node.variations[0].move)
node = node.variations[0]
# headers["Result"]: {"0-1", "1-0", "1/2-1/2"}
# result: {-1, 0, 1}
# Parse result from header
white_score = game.headers["Result"].split("-")[0].split("/")
if len(white_score) == 1:
result = 2 * int(white_score[0]) - 1
else:
result = 0
state = state_from_board(board).reshape(
(1, NUM_COLORS * NUM_PIECES, NUM_ROWS, NUM_COLS))
reward = np.array([(GAMMA**moves_remaining) * result])
self.idx_game += 1
yield state, reward