def parser(file, elo=400, draws=False, time_control=True):
pgn = open(file)
games = []
results = []
offsets = []
counter = 0
while True:
offset = pgn.tell()
headers = chess.pgn.read_headers(pgn)
if headers is None:
break
if headers.get("Result") not in ["1-0", "0-1", "1/2-1/2"]:
continue
if not draws:
if headers.get("Result") == "1/2-1/2":
continue
elo_white = headers.get("WhiteElo")
elo_black = headers.get("BlackElo")
if elo_white is None or not elo_white.isnumeric():
elo_white = 0
if elo_black is None or not elo_black.isnumeric():
elo_black = 0
time_control = headers.get("TimeControl")
if time_control:
if not time_control[0:4].isnumeric():
if not time_control[0:3].isnumeric():
continue
if int(time_control[0:3]) < 180:
continue
if int(elo_black) >= elo or int(elo_white) >= elo:
offsets.append(offset)
result = headers.get("Result")
if result == "1-0":
results.append("white")
elif result == "0-1":
results.append("black")
else:
print(result)
results.append("draw")
counter += 1
if counter > 50000:
break
for offset in offsets:
pgn.seek(offset)
games.append(chess.pgn.read_game(pgn))
return games, results
def read_games(pgn_file, n_matches): pgn = open(pgn_file) g_count = 0 games = [] while pgn.tell() != os.fstat(pgn.fileno()).st_size: game = chess.pgn.read_game(pgn) games.append(game) g_count += 1 if g_count == n_matches: break print >> sys.stderr, "Total ",g_count,"games has been read" return games
def filter_pgn_file_for_headers(
pgn: TextIO,
filters: Dict[str, List[Any]] = {},
) -> Generator[chess.pgn.Headers, None, None]:
while True:
_ = pgn.tell()
headers = chess.pgn.read_headers(pgn)
if headers is None:
break
for key, vals in filters.items():
if headers.get(key) not in vals:
break
else:
yield headers
def gen(pgn_file_path):
with open(pgn_file_path) as pgn:
# Start from random position in the file
pgn.seek(0, 2)
pgn.seek(random.randint(0, pgn.tell()))
chess.pgn.read_headers(pgn)
while True:
game = chess.pgn.read_game(pgn)
if not game:
pgn.seek(0)
continue
'''
result_header = game.headers['Result']
game_value_for_white = 0
if result_header == '*':
continue
elif result_header == '1-0':
game_value_for_white = 1
elif result_header == '0-1':
game_value_for_white = -1
else:
game_value_for_white = 0
'''
board = game.board()
for node in game.mainline():
board.push(node.move)
eval = node.eval()
if not eval:
break
eval = eval.pov(not board.turn).score()
if not eval:
continue
try:
X = get_halfkp_indeces(board)
except:
print(f'Would have crashed: {board}')
continue
# y = game_value_for_white if board.turn == chess.WHITE else -game_value_for_white
# y = eval if board.turn == chess.WHITE else -eval
yield (X[0], X[1]), eval / 100
def get_offsets(pgn, limit):
offsets = []
for i in itertools.count():
if i % 10000 == 0:
print('{:>10} {:>10}'.format(i, len(offsets)))
offset = pgn.tell()
headers = chess.pgn.read_headers(pgn)
if headers is None:
break
if headers['TimeControl'] != time_control:
continue
if headers["WhiteElo"] == "?":
continue
if headers["BlackElo"] == "?":
continue
if limit and len(offsets) >= limit:
break
offsets.append(offset)
return offsets
def filter_pgn_file_for_games(
pgn: TextIO,
filters: Dict[str, List[Any]] = {},
verbose_period: Optional[int] = None,
) -> Generator[chess.pgn.Game, None, None]:
count = 0
while True:
offset = pgn.tell()
headers = chess.pgn.read_headers(pgn)
if headers is None:
break
for key, vals in filters.items():
if headers.get(key) not in vals:
break
else:
pgn.seek(offset)
game = chess.pgn.read_game(pgn)
count += 1
if verbose_period is not None and count % verbose_period == 0:
print(f"{count} games processed")
yield game
def get_games_from_file(filename):
"""
:param str filename: file containing the pgn game data
:return list(pgn.Game): chess games in that file
"""
pgn = open(filename, errors='ignore')
offsets = []
while True:
offset = pgn.tell()
headers = chess.pgn.read_headers(pgn)
if headers is None:
break
offsets.append(offset)
n = len(offsets)
print(f"found {n} games")
games = []
for offset in offsets:
pgn.seek(offset)
games.append(chess.pgn.read_game(pgn))
return games
regexFen = regexFenPosition[:-1] + r' [wb] (?:(?:K?Q?k?q?)|-) (?:(?:[a-h][1-8])|-) \d+ \d+$'
def fix(filename):
subprocess.call(['fastfen', './fens/'+filename])
if __name__ == '__main__':
pathPgn = './output.pgn'
if sys.argv[1:]:
pathPgn = sys.argv[1]
print "opening PGN: %s" % pathPgn
pgn = open(pathPgn)
lineCur = 1
lineToOffset = {}
while 1:
lineToOffset[lineCur] = pgn.tell()
if not pgn.readline():
break;
lineCur += 1
pgn.seek(0)
print lineToOffset
sys.exit(-1)
gameNum = 1
while 1:
game = chess.pgn.read_game(pgn)
fen = game.headers["FEN"]
print "offset:%d, game:%d, fen:%s" % (pgn.tell(), gameNum, fen)
def read_game(self, idx):
self.env.reset()
self.black = ChessPlayer(self.config, self.model)
self.white = ChessPlayer(self.config, self.model)
files = find_pgn_files(self.config.resource.play_data_dir)
if len(files) > 0:
random.shuffle(files)
filename = files[0]
pgn = open(filename, errors='ignore')
size = os.path.getsize(filename)
pos = random.randint(0, size)
pgn.seek(pos)
line = pgn.readline()
offset = 0
# Parse game headers.
while line:
if line.isspace() or line.startswith("%"):
line = pgn.readline()
continue
# Read header tags.
tag_match = TAG_REGEX.match(line)
if tag_match:
offset = pgn.tell()
break
line = pgn.readline()
pgn.seek(offset)
game = chess.pgn.read_game(pgn)
node = game
result = game.headers["Result"]
actions = []
while not node.is_end():
next_node = node.variation(0)
actions.append(node.board().uci(next_node.move))
node = next_node
pgn.close()
k = 0
observation = self.env.observation
while not self.env.done and k < len(actions):
if self.env.board.turn == chess.BLACK:
action = self.black.sl_action(observation, actions[k])
else:
action = self.white.sl_action(observation, actions[k])
board, info = self.env.step(action)
observation = board.fen()
k += 1
self.env.done = True
if not self.env.board.is_game_over() and result != '1/2-1/2':
self.env.resigned = True
if result == '1-0':
self.env.winner = Winner.white
elif result == '0-1':
self.env.winner = Winner.black
else:
self.env.winner = Winner.draw
self.finish_game()
self.save_play_data(write=idx % self.config.play_data.nb_game_in_file == 0)
self.remove_play_data()
return self.env
def fix(filename):
subprocess.call(['fastfen', './fens/' + filename])
if __name__ == '__main__':
pathPgn = './output.pgn'
if sys.argv[1:]:
pathPgn = sys.argv[1]
print "opening PGN: %s" % pathPgn
pgn = open(pathPgn)
lineCur = 1
lineToOffset = {}
while 1:
lineToOffset[lineCur] = pgn.tell()
if not pgn.readline():
break
lineCur += 1
pgn.seek(0)
print lineToOffset
sys.exit(-1)
gameNum = 1
while 1:
game = chess.pgn.read_game(pgn)
fen = game.headers["FEN"]
print "offset:%d, game:%d, fen:%s" % (pgn.tell(), gameNum, fen)
m = re.match(regexFen, fen)
def read(filename, data):
pgn = open(filename)
cnt = 0
while True:
cnt += 1
if cnt % 10000 == 0:
print(cnt)
offset = pgn.tell()
headers = chess.pgn.read_headers(pgn)
if headers is None:
break
point = 0
if headers['Result'] == '1-0':
point = 1
if headers['Result'] == '1/2-1/2':
point = 0.5
date = headers['UTCDate']
if date not in data:
data[date] = {}
data[date]['games'] = [1, 0]
data[date]['traps'] = [[0, 0], [0, 0], [0, 0], [0, 0], [0, 0],
[0, 0]]
"""
0: Oh no my queen, Stafford
1: Oh no my knight, Stafford
2: Stafford: 5.e5 Ne4 6.d4 Qh4 7.g3 Ng3
3: Punishing natural development, Stafford
4: Punishing capture hxg4, Stafford
5: Englund gambit trap
"""
else:
data[date]['games'][0] += 1
data[date]['games'][1] += point
opening = headers["Opening"]
if opening not in data[date]:
data[date][opening] = [1, 0]
else:
data[date][opening][0] += 1
data[date][opening][1] += point
if opening == 'Englund Gambit' or opening == 'Russian Game: Stafford Gambit':
pgn.seek(offset)
mygame = chess.pgn.read_game(pgn)
for i in range(10):
if mygame.next() != None:
mygame = mygame.next()
if mygame.board().fen(
) == 'rnbqk2r/ppp1Pppp/8/8/8/5N2/PPP1PbPP/RNBQKB1R w KQkq - 0 6':
data[date]['traps'][5][0] += 1
data[date]['traps'][5][1] += point
print(date, data[date]['traps'])
mygame = mygame.game()
for i in range(13):
if mygame.next() != None:
mygame = mygame.next()
if mygame.board().fen(
) == 'r1bBk2r/ppp2ppp/2p5/2b5/4n3/3P4/PPP2PPP/RN1QKB1R b KQkq - 0 7':
data[date]['traps'][0][0] += 1
data[date]['traps'][0][1] += point
print(date, data[date]['traps'])
mygame = mygame.game()
for i in range(12):
if mygame.next() != None:
mygame = mygame.next()
if mygame.board().fen(
) == 'r1bqk2r/ppp2ppp/2p5/2b1P3/4n3/3P4/PPP2PPP/RNBQKB1R w KQkq - 1 7':
data[date]['traps'][1][0] += 1
data[date]['traps'][1][1] += point
print(date, data[date]['traps'])
mygame = mygame.game()
for i in range(14):
if mygame.next() != None:
mygame = mygame.next()
if mygame.board().fen(
) == 'r1b1kb1r/ppp2ppp/2p5/4P3/3P3q/6n1/PPP2P1P/RNBQKB1R w KQkq - 0 8':
data[date]['traps'][2][0] += 1
data[date]['traps'][2][1] += point
print(date, data[date]['traps'])
mygame = mygame.game()
for i in range(16):
if mygame.next() != None:
mygame = mygame.next()
if mygame.board().fen(
) == 'r1b1k2r/ppp2ppp/2p5/2b5/2B1P2q/2N4P/PPPP1nP1/R1BQ1RK1 w kq - 0 9':
data[date]['traps'][3][0] += 1
data[date]['traps'][3][1] += point
print(date, data[date]['traps'])
mygame = mygame.game()
for i in range(17):
if mygame.next() != None:
mygame = mygame.next()
if mygame.board().fen() == 'r1b1k2r/ppp2pp1/2p5/2b4p/3qP1P1/2N5/PPPPBPP1/R1BQ1RK1 b kq - 0 9' or \
mygame.board().fen() == 'r1b1k2r/ppp2pp1/2pq4/2b4p/4P1P1/3P4/PPP1BPP1/RNBQ1RK1 b kq - 0 9':
data[date]['traps'][4][0] += 1
data[date]['traps'][4][1] += point
print(date, data[date]['traps'])
return data
def get_end_offset(self):
with open(self.filename,'r') as pgn:
pgn.seek(0,os.SEEK_END)
end_offset = pgn.tell()
print("determined end offset: "+str(end_offset))
return end_offset
def get_end_offset(self):
with open(self.filename, 'r') as pgn:
pgn.seek(0, os.SEEK_END)
end_offset = pgn.tell()
return end_offset