def check_london(data_2300, data_2600):
files = ['../databases/london-2300.pgn', '../databases/london-2600.pgn']
totals = [data_2300, data_2600]
for i in range(2):
count = np.zeros(len(years))
with open(files[i], 'r') as pgn:
game = p.read_game(pgn)
num_games = 0
while game:
ply = 1
for move in game.mainline_moves():
uci = move.uci()
if uci == 'c1f4':
year = int(game.headers['Date'].split('.')[0])
count[year_index(year)] += 1
break
ply += 1
if ply > 5:
break
num_games += 1
if num_games % 1000 == 0:
print(num_games)
game = p.read_game(pgn)
if not game:
game = p.read_game(pgn)
print(num_games)
total = np.sum(totals[i][:, total_d4], axis=1)
freq = freq_smooth(count, total)
plt.plot(years, freq, label=levels[i])
plt.axis([year_start - 1, year_end + 1, 0, 30])
plt.grid()
plt.legend()
plt.title('London')
plt.show()
def pgn_generator():
with open('sources/ccrl_stockfish.pgn', 'r', encoding='Latin-1') as file:
game = pgn.read_game(file)
while game is not None:
node = game.variations[0]
move = 1
if 'Stockfish' in game.headers['Black']:
node = node.variations[0]
move += 1
while not node.is_end():
comment = node.comment
score = re.findall(' [-+]\d*.\d*', comment)
depth = re.findall('/\d*', comment)
result = get_result_simple(game.headers['Result'])
if score and depth:
yield move, float(score[0]), result
# we are skipping every other move
node = node.variations[0]
if node.is_end():
break
else:
node = node.variations[0]
move += 2
game = pgn.read_game(file)
def parseData_value(PGNFileDir, npzFilePath, boardShape=(8, 8)):
if os.path.exists(npzFilePath):
print("error! the Path has existed a npz file!!")
return
pgnfileNames = os.listdir(PGNFileDir)
pgnfilePaths = list(map(lambda x: PGNFileDir + "/" + x, pgnfileNames))
labels = []
features = []
for path in pgnfilePaths:
pgnFile = open(path, encoding="utf-8-sig")
game = pgn.read_game(pgnFile)
if game == None:
print("empty pgn file")
continue
else:
random.seed()
while game is not None:
result = game.headers.get("Result")
if result == "*":
game = pgn.read_game(pgnFile)
continue
root = game
board = game.board()
lens = len(list(game.main_line()))
if lens == 0:
print(game)
print("no move game")
game = pgn.read_game(pgnFile)
continue
index = random.randint(0, lens)
nextNode = root
for x in range(index):
nextNode = nextNode.variation(0)
move = nextNode.move
board.push(move)
else:
feature_board = get_location_array_chw(board, boardShape)
feature_turn = bool2array(board.turn, boardShape)
feature_board.append(feature_turn)
features.append(np.array(feature_board))
label = -1.0
if result == "1-0" and board.turn == True:
label = 1.0
if result == "0-1" and board.turn == False:
label = 1.0
if result == "1/2-1/2":
label = 0.0
labels.append(np.array([label]))
game = pgn.read_game(pgnFile)
print(feature_turn)
else:
# save2npzfile(npzFilePath, np.array(features), np.array(labels))
pass
return
def get_game(PGNFilePath):
pgnFile = open(PGNFilePath)
games = []
game = pgn.read_game(pgnFile)
if game is None:
print("empty pgnfiles")
return
else:
while game is not None:
games.append(game)
game = pgn.read_game(pgnFile)
return games
def playable(self):
try:
from StringIO import StringIO
except ImportError:
from io import StringIO
return read_game(StringIO(" ".join(self.pgn)))
def get_all_games_states(
pgn_file: TextIO, games_to_get: int, return_legal_moves: bool,
show_progress: bool
) -> Union[Tuple[List[np.ndarray], LegalMovesT], List[np.ndarray]]:
"""Extracts a list for each game with a list of states"""
all_states, all_legal_moves = [], []
iterator = tqdm(
range(games_to_get)) if show_progress else range(games_to_get)
for _ in iterator:
game = pgn.read_game(pgn_file)
assert not game.errors
game_states = get_single_games_states(
game=game, return_legal_moves=return_legal_moves)
if return_legal_moves:
game_states, game_legal_moves = game_states
all_legal_moves.append(game_legal_moves)
game_states = np.array(game_states)
all_states.append(game_states)
if return_legal_moves:
return all_states, all_legal_moves
else:
return all_states
def pgn_to_board_string_list(pgn_path, trim_equal=True):
# Loads pgn file and converts to board string split by whether it is players move or not
username = extract_username(pgn_path)
with open(pgn_path, 'r', encoding='utf-8') as f:
game = pgn.read_game(f)
board = game.board()
before_move = []
after_move = []
user_turn = game.headers["White"].lower() == username
first_turn = True
# print(f'User has first turn: {user_turn}')
for m in game.mainline_moves():
board_str = str(board)
if user_turn:
before_move.append(board_str)
elif not first_turn:
after_move.append(board_str)
user_turn = not user_turn
board.push(m)
first_turn = False
if trim_equal:
before_move = before_move[:len(after_move)]
return before_move, after_move
def annotated_sample_generator_engine960():
"""
Generates annotated training data. Uses engines960.pgn games dataset.
Generator yields a pair of (samples, labels) tensors
where samples represents game positions like the following:
[[-0.5,-0.3,-0.35,-0.8,-1.0,-0.35,-0.3,-0.5,
-0.1,-0.1, -0.1,-0.1,-0.1, -0.1,-0.1,-0.1,
0, 0, 0, 0, 0, 0, 0, 0,
...
0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1,
0.5, 0.3, 0.35, 0.8, 1.0, 0.35, 0.3, 0.5],[..]..]
and labels of the form:
[0.24, 0.18, ..]
"""
pgn = open(os.path.join(PROJECT_ROOT_DIR, "data/pgn/engines960.pgn"))
# game_labels = load_labels()
while True:
try:
game = read_game(pgn)
if not game:
break
labels = game_labels_from_coments(game)
samples = sample_game(game)
yield samples, labels
except:
pass
def main():
'''
Read into API and save PGNs
'''
for r, dirs, files in os.walk(args.dir):
files = list(filter(lambda x: x.endswith('pgn'), files))
files = list(map(lambda x: os.path.join(r, x), files))
break
## Now, load the games in PGN format
games = []
for _file in files:
with open(_file, 'r') as fi:
while True:
game = pgn.read_game(fi)
if game is None:
break
games.append(game)
np.random.shuffle(games)
N = len(games)
traingames = games[:int(0.9 * N)]
valgames = games[int(0.9 * N):]
with open(os.path.join(args.dir, 'train.pgn'), 'w') as fi:
for game in traingames:
print(game, file=fi, end="\n\n")
with open(os.path.join(args.dir, 'val.pgn'), 'w') as fi:
for game in valgames:
print(game, file=fi, end="\n\n")
async def show_board(msg):
contenu = msg.content.split(' ')
if len(contenu) == 1:
await msg.channel.send("Pas d'id de partie donné")
return
id_partie = 0
try:
id_partie = int(contenu[1])
except ValueError:
await msg.channel.send("Pas le bon format d'identifiant")
return
em = discord.Embed(title='Partie ' + str(id_partie))
modele_requete = "SELECT pgn FROM games WHERE id_partie = '{0}';"
requete = modele_requete.format(id_partie)
cdesc = psycopg2.connect(**params_db)
cursor = cdesc.cursor()
cursor.execute(requete)
(pgn_, ) = cursor.fetchone()
pgn_strio = io.StringIO(pgn_)
game = pgn.read_game(pgn_strio)
board = game.board()
for move in game.mainline_moves():
board.push(move)
fen = board.fen().split()[0].strip()
addr = "https://backscattering.de/web-boardimage/board.png?fen=" + fen
em.set_image(url=addr)
await msg.channel.send(embed=em)
def create_epd_set(pgnfile, writefile):
"""
method to generate a set of epd strings fom a pgn dataset.
"""
dset = open(pgnfile)
stop = False
epd_set = []
i = 0
while not stop and i < 500000:
i += 1
try:
game = pgn.read_game(dset)
if game is None:
break
node = game
while not node.is_end():
node = node.variation(0)
b = node.board().epd()
epd_set.append(b)
except Exception as e:
print e
break
if i % 1000 == 0:
print(i)
print epd_set[:100]
with open(writefile, 'w') as f:
cp.dump(epd_set, f)
def main(pgn=None):
times = list()
wins = losses = draws = 0
for i in range(10):
try:
with open(pgn) as pgn:
game = pgn_reader.read_game(pgn)
board = game.board()
except Exception:
board = Board()
white_player = EnginePlayer("White", depth=.01)
black_player = EnginePlayer("Black", depth=.1)
start = time.time()
print("Starting game", i + 1)
while True:
# print(board)
if board.is_game_over():
if is_white_win(board.result()):
wins = wins + 1
elif is_black_win(board.result()):
losses = losses + 1
else:
draws = draws + 1
handle_score(board)
break
white_turn = board.turn
if white_turn:
move = white_player.get_next_move(board)
else:
move = black_player.get_next_move(board)
if move is "-":
exit()
board.push(move)
white_player.clean()
black_player.clean()
end = time.time()
diff = (end - start)
print("Game lasted {:.0f} seconds\n".format(diff))
times.append(diff)
engines.clean_all()
print("Average time per game", numpy.average(times))
print("Standard deviation for all games", numpy.std(times))
print("Total wins", wins)
print("Total losses", losses)
print("Total draws", draws)
def parse_pgn(args):
basename = path.splitext(path.basename(args.source_file))[0]
output_file = path.join(args.output_dir, f"{basename}_uci.txt")
game_counter = 0
with open(output_file,
'w') as output_f, open(args.source_file) as source_f:
while True:
try:
game = pgn.read_game(source_f)
board = game.board()
if game is None:
break
move_list = []
for move in game.mainline_moves():
move_list.append(board.uci(move))
board.push(move)
output_f.write(" ".join(move_list) + "\n")
game_counter += 1
if game_counter % 1e4 == 0:
logger.info(f"{game_counter} games processed")
if game_counter >= args.max_games:
break
except (ValueError, UnicodeDecodeError) as e:
pass
logger.info(f"Extracted {game_counter} games at {output_file}")
def createhistory(pgn):
historyobj = None
try:
pgnio = io.StringIO(pgn)
game = read_game(pgnio)
board = game.board()
positioninfos = []
pinfo = {
"fen": board.fen()
}
addpositioninfo(board, pinfo)
positioninfos.append(pinfo)
for move in game.main_line():
genboard = board.copy()
board.push(move)
pinfo = {
"fen": board.fen()
}
addpositioninfo(board, pinfo, move, genboard)
positioninfos.append(pinfo)
historyobj = {
"positioninfos": positioninfos,
"pgn": pgn,
"uci_variant": board.uci_variant,
"chess960": board.chess960
}
return ( historyobj , "game history created ok" )
except:
traceback.print_exc(file=sys.stderr)
return ( None , "! create game history failed" )
def serve_gif(gameid):
meta = request.args.get('meta')
result = requests.get(f'https://lichess.org/game/export/{gameid}')
data = requests.get(f'https://lichess.org/api/game/{gameid}').json()
game = read_game(StringIO(result.text))
size = 360
tempfile = TemporaryFile()
with imageio.get_writer(tempfile, mode='I', format='gif', fps=1) as writer:
node = game
while not node.is_end():
nextNode = node.variation(0)
board_svg = chess.svg.board(node.board(),
coordinates=False,
flipped=False,
size=size,
style=style)
board_png = imageio.imread(cairosvg.svg2png(bytestring=board_svg))
writer.append_data(board_png)
node = nextNode
if meta:
splash = create_splash(size, data)
writer.append_data(splash)
tempfile.seek(0)
return send_file(tempfile, mimetype='image/gif')
def analyseGame(self, game: Game, colour: Colour,
nodes: int) -> Opt[AnalysedGame]:
gameLen = len(game.pgn)
if gameLen < 40 or gameLen > 120:
logging.warning(f'game too long/short to analyse ({gameLen} plys)')
return None
elif game.emts is None:
logging.warning(f'game has no emts')
return None
analysedMoves = []
try:
playableGame = read_game(StringIO(" ".join(game.pgn)))
except ValueError:
return None
node = playableGame
self.engine.ucinewgame()
while not node.is_end():
logging.info(f'analysing position\n{node.board()}\n')
nextNode = node.variation(0)
if colour == node.board(
).turn: ## if it is the turn of the player of interest
self.engine.setoption({'multipv': 5})
self.engine.position(node.board())
self.engine.go(nodes=nodes)
analyses = list([
Analysis(pv[1][0].uci(),
EngineEval(engineEval[1].cp, engineEval[1].mate))
for engineEval, pv in zip(
self.infoHandler.info['score'].items(),
self.infoHandler.info['pv'].items())
])
self.engine.setoption({'multipv': 1})
self.engine.position(nextNode.board())
self.engine.go(nodes=nodes)
engineEval = EngineEval(
self.infoHandler.info['score'][1].cp,
self.infoHandler.info['score'][1].mate).inverse(
) # flipped because analysing from other player side
moveNumber = node.board().fullmove_number
analysedMoves.append(
AnalysedMove(uci=node.variation(0).move.uci(),
move=moveNumber,
emt=game.emts[EngineTools.ply(
moveNumber, colour)],
engineEval=engineEval,
analyses=analyses))
node = nextNode
playerId = game.white if colour else game.black
return AnalysedGame.new(game.id, colour, playerId, analysedMoves)
def load_games(self, games_filename=None):
if not games_filename and not self.games_filename:
print('Missing a filename for the pgn to load')
return
if games_filename:
self.set_games_filename(games_filename)
pgn = self.games_filename
with open(pgn) as filename:
games = {}
game = None
while (game or len(games)
== 0) and len(games) < self.max_number_of_games_to_analyse:
try:
game = read_game(filename)
if game is not None:
variant = game.headers['Variant']
if variant == 'Standard':
game_id = get_game_id(game)
games[game_id] = {}
games[game_id]['raw'] = game
except ValueError as e:
print(len(games), e)
self.games = games
return self.games
def annotated_sample_generator_labels_in_csv():
"""
Generates annotated training data. Uses games.pgn games dataset.
Each game has an event number. Labels extracted from labels.cs
Generator yields a pair of samples, labels tensors
where samples represents game positions like the following:
[[-0.5,-0.3,-0.35,-0.8,-1.0,-0.35,-0.3,-0.5,
-0.1,-0.1, -0.1,-0.1,-0.1, -0.1,-0.1,-0.1,
0, 0, 0, 0, 0, 0, 0, 0,
...
0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1,
0.5, 0.3, 0.35, 0.8, 1.0, 0.35, 0.3, 0.5],[..]..]
and labels of the form:
[0.24, 0.18, ..]
"""
pgn = open("data/pgn/games.pgn")
game_labels = load_labels()
while True:
try:
game = read_game(pgn)
if not game:
break
event = game.headers["Event"]
labels = game_labels[event]
labels = labels.split(' NA')[0]
labels = list(
map(lambda x: float(x) / 1000,
labels.split(' ')[::2]))
samples = sample_game(game)
yield samples[:len(labels)], labels
except:
pass
def main():
morphy = open("data/aggressive/Morphy.pgn", encoding="utf-8-sig")
headers = [
"Event", "Site", "Date", "Round", "White", "Black", "Result",
"WhiteElo", "BlackElo", "ECO", "Aggro"
]
df = pd.DataFrame(columns=headers)
g = c_pgn.read_game(morphy)
while g is not None:
d = dict(g.headers)
d["Aggro"] = 1
df = df.append(d, ignore_index=True)
g = c_pgn.read_game(morphy)
print(df.head())
async def fetch_game(game_id):
game = current_games[game_id]
pgn_string = io.StringIO(game['pgn'])
game = pgn.read_game(pgn_string)
board = game.board()
for move in game.mainline_moves():
board.push(move)
return board
def parse_pgn(pgn_file):
with open('data.csv', 'w') as csv_file:
csv_writer = csv.writer(csv_file, delimiter=';')
csv_writer.writerow(['POSITION', 'RESULT'])
i = 0
while i < 100:
game = pgn.read_game(pgn_file)
write_game_row(csv_writer, game)
i += 1
def load_game(channel_or_nick):
"""
Given a `channel_or_nick`, will load the active game from mongo.
"""
game_doc = find_game(channel_or_nick)
if game_doc:
pgn_string = StringIO(game_doc['pgn'])
return pgn.read_game(pgn_string)
def main(args):
np.random.seed(args.seed)
import chess.engine
engine = chess.engine.SimpleEngine.popen_uci(args.stockfish_path)
games = open(args.games_file)
idxs = []
labels = []
values = []
net_wins = 0
i = 0
bar = tqdm(total=args.num_games)
imbalance = max(args.num_games // 100, 100)
game = read_game(games)
while i < args.num_games and game is not None:
if i % 10000 == 0:
tqdm.write(f'# board positions: {len(idxs)}')
if i % args.save_interval == 0 and i != 0:
np.savez(args.boards_file, idxs=np.array(idxs),
labels=np.array(labels))
label = get_label(game)
# if label is not None and abs(net_wins + (label * 2) - 1) < imbalance:
board = game.board()
moves = list(game.mainline_moves())
num_boards = min(args.num_samples, len(moves))
move_idxs = set(np.random.choice(range(len(moves)), num_boards,
replace=False))
# net_wins += (label * 2 - 1) * num_boards
for j, move in enumerate(moves):
board.push(move)
if j in move_idxs:
idxs.append(get_idxs(board))
info = engine.analyse(board, chess.engine.Limit(
time=args.time_limit))
labels.append(label)
values.append(info['score'].white().score())
i += 1
bar.update(1)
game = read_game(games)
print(abs(len(labels) // 2 - sum(labels)))
print(len(labels))
bar.close()
np.savez(args.boards_file, idxs=np.array(idxs), labels=np.array(labels))
def _get_positions(cls, file_name: str) -> List[str]:
positions: List[str] = []
num_games = 0
log_after = 10000
start_time = time()
local_start_time = time()
with open(file_name, encoding="UTF-8") as game_file:
game = read_game(game_file)
while game is not None:
positions += cls._parse_game(game)
game = read_game(game_file)
num_games += 1
if num_games % log_after == 0:
cls._log_extracting_progress(file_name, start_time, local_start_time,
num_games, log_after, len(positions))
local_start_time = time()
cls._log_extracting_progress(file_name, start_time, local_start_time, num_games, None,
len(positions))
return positions
def print_board_seq_from_pgn(path):
with open(path, encoding='utf-8') as f:
game = pgn.read_game(f)
board = game.board()
for m in game.mainline_moves():
board.push(m)
print(board)
print('-' * 15)
def get_games_from_pgn_files():
"Parse PGN files in the current directory and return a list of parsed game objects"
game_list = []
for file in glob.glob("resources/input/*.pgn"):
file_handle = open(file)
while True:
game = pgn.read_game(file_handle)
if game is None:
break
game_list.append(game)
return game_list
def newGame(self, newGamePath=None):
if newGamePath:
with open(newGamePath) as pgnFile:
self.game = pgn.read_game(pgnFile)
print('opened', newGamePath)
else:
print('new game')
self.fileHandle = None
self.game = pgn.Game()
self.updateCurrent(self.game.root())
self.positionChanged.emit(self.current)
def parse_pgn(input_path, output_path):
pgn_file = open(input_path)
game = pgn.read_game(pgn_file)
index = 0
if not os.path.exists(output_path):
os.makedirs(output_path)
while game:
pgn2board(game, '{0}/{1}'.format(output_path, index))
index += 1
def read_data(file,limit):
games = []
pgn = open(file)
# with open(file) as pgn:
for i in tqdm(range(limit)):
game = ch.read_game(pgn)
if game.headers['Event'] != 'FICS rated blitz game': continue
if game.headers['TimeControl'] != '180+0': continue
# for consistency, we only look at blitz games with time control value of 180+0
games.append(parse_header(game))
pgn.close()
return games
def test(PGNFileDir, npzFilePath, boardShape=(8, 8)):
pgnfileNames = os.listdir(PGNFileDir)
pgnfilePaths = list(map(lambda x: PGNFileDir + "/" + x, pgnfileNames))
labels = []
features = []
for path in pgnfilePaths:
pgnFile = open(path, encoding="utf-8-sig")
game = pgn.read_game(pgnFile)
if game == None:
print("empty pgn file")
continue
else:
random.seed()
while game is not None:
result = game.headers.get("Result")
if result == "*":
game = pgn.read_game(pgnFile)
continue
root = game
board = game.board()
lens = len(list(game.main_line()))
if lens == 0:
print(game)
print("no move game")
game = pgn.read_game(pgnFile)
continue
index = random.randint(0, lens)
nextNode = root
for x in range(index):
nextNode = nextNode.variation(0)
move = nextNode.move
board.push(move)
else:
print(board)
game = pgn.read_game(pgnFile)
print(len(labels))
return
def find_game(channel_or_nick):
"""
Return the first, and hopefully only, active game, by scanning
each game's Result header, or None if there are no active games.
"""
games = db.chess.find({
'opponent': channel_or_nick.strip('#'),
})
for game_doc in games:
pgn_string = StringIO(game_doc['pgn'])
pgn_game = pgn.read_game(pgn_string)
if pgn_game.headers['Result'] == '*':
return game_doc
def choose_game(pgnpath):
pgn = open(pgnpath, "r")
games_in_pgn = []
while True:
offset = pgn.tell()
header = read_headers(pgn)
if header is None:
break
game_desc = f"{header['White']} vs {header['Black']}, {header['Result']}"
games_in_pgn.append((game_desc, offset))
chosen = games_in_pgn[0]
if len(games_in_pgn) > 1:
for i, (head, peek) in enumerate(games_in_pgn):
print(f"{i+1:3d}: {head}")
chosen = games_in_pgn[int(input("Choose: ")) - 1]
pgn.seek(chosen[1])
return read_game(pgn), chosen[0]
def process(self):
while True:
game = pgn.read_game(self.pgn_file)
if game is None: # It's end of the pgn file
break
# This is a new game
our_side = self.get_our_side(game)
if our_side == 0:
continue
if 'ccrl' in game.headers["Event"].lower():
event = 'ccrl'
if 'tcec' in game.headers["Event"].lower():
event = 'tcec'
winning_side = self.get_winning_side(game)
pos1 = game.end()
# We are reading 2 consecutive moves to get our eval and opponent's eval
pos2 = pos1.parent
counter = 0
while pos1 is not None and pos2 is not None and counter <= 20:
if event == 'ccrl':
ev1 = self.find_evaluation_in_comment(pos1, 'ccrl')
ev2 = self.find_evaluation_in_comment(pos2, 'ccrl')
elif event == 'tcec':
ev1 = self.find_evaluation_in_comment(pos1, 'tcec')
ev2 = self.find_evaluation_in_comment(pos2, 'tcec')
turn1 = 1 if pos1.board().turn else -1
if ev1 is not None and ev2 is not None:
our_eval = ev1 if turn1 == our_side else ev2
opp_eval = ev2 if turn1 == our_side else ev1
if self.opponent_is_right(winning_side, our_side, our_eval, opp_eval):
self.epd_file.write(pos1.board().epd() + '\n')
self.epd_file.write(str(winning_side * 123) + '\n')
# get previous position
pos1 = pos2
pos2 = pos1.parent
counter += 1
# for diagnostics
#num_games /= 75
early_boards = []
late_boards = []
results = []
##
## won or loss games with ply > 10
##
with open ('PGN_files\games.pgn') as games:
for _ in range(num_games):
g = pgn.read_game(games)
if not g.headers['Result'] == '1-0' and not g.headers['Result'] == '0-1':
continue
plys = int(g.headers['PlyCount'])
if not plys >10:
continue
if g.headers['Result'] == '1-0':
results.append(1)
else:
results.append(-1)
def fetch_game(self):
self._g = pgn.read_game(self._fp)
return self._g is not None
