def init_scoutfish(self):
""" Create/open .scout database index file to help querying
using scoutfish from https://github.com/mcostalba/scoutfish
"""
if scoutfish_path is not None and self.path and self.size > 0:
try:
if self.progressbar is not None:
from gi.repository import GLib
GLib.idle_add(self.progressbar.set_text,
_("Creating .scout index file..."))
self.scoutfish = Scoutfish(engine=(scoutfish_path, ))
self.scoutfish.open(self.path)
scout_path = os.path.splitext(self.path)[0] + '.scout'
if getmtime(self.path) > getmtime(scout_path):
self.scoutfish.make()
except OSError as err:
self.scoutfish = None
log.warning("Failed to sart scoutfish. OSError %s %s" %
(err.errno, err.strerror))
except pexpect.TIMEOUT:
self.scoutfish = None
log.warning("scoutfish failed (pexpect.TIMEOUT)")
except pexpect.EOF:
self.scoutfish = None
log.warning("scoutfish failed (pexpect.EOF)")
def init_scoutfish(self):
# Create/open .scout database index file to help querying
# using scoutfish from https://github.com/mcostalba/scoutfish
if scoutfish_path is not None and self.path and self.size > 0:
try:
if self.progressbar is not None:
self.progressbar.set_text("Creating .scout index file...")
self.scoutfish = Scoutfish(engine=scoutfish_path)
self.scoutfish.open(self.path)
scout_path = os.path.splitext(self.path)[0] + '.scout'
if getmtime(self.path) > getmtime(scout_path):
self.scoutfish.make()
except OSError as err:
self.scoutfish = None
log.debug("Failed to sart scoutfish. OSError %s %s" %
(err.errno, err.strerror))
except pexpect.TIMEOUT:
self.scoutfish = None
print("scoutfish failed (pexpect.TIMEOUT)")
except pexpect.EOF:
self.scoutfish = None
print("scoutfish failed (pexpect.EOF)")
def init_scoutfish(self):
""" Create/open .scout database index file to help querying
using scoutfish from https://github.com/mcostalba/scoutfish
"""
if scoutfish_path is not None and self.path and self.size > 0:
try:
if self.progressbar is not None:
self.progressbar.set_text("Creating .scout index file...")
self.scoutfish = Scoutfish(engine=(scoutfish_path, ))
self.scoutfish.open(self.path)
scout_path = os.path.splitext(self.path)[0] + '.scout'
if getmtime(self.path) > getmtime(scout_path):
self.scoutfish.make()
except OSError as err:
self.scoutfish = None
log.warning("Failed to sart scoutfish. OSError %s %s" % (err.errno, err.strerror))
except pexpect.TIMEOUT:
self.scoutfish = None
log.warning("scoutfish failed (pexpect.TIMEOUT)")
except pexpect.EOF:
self.scoutfish = None
log.warning("scoutfish failed (pexpect.EOF)")
class PGNFile(ChessFile):
def __init__(self, handle, progressbar):
ChessFile.__init__(self, handle)
self.handle = handle
self.progressbar = progressbar
self.pgn_is_string = isinstance(handle, StringIO)
if self.pgn_is_string:
self.games = [self.load_game_tags(), ]
else:
self.skip = 0
self.limit = 100
self.order_col = game.c.offset
self.is_desc = False
self.reset_last_seen()
# filter expressions to .sqlite .bin .scout
self.tag_query = None
self.fen = None
self.scout_query = None
self.init_tag_database()
self.scoutfish = None
self.init_scoutfish()
self.chess_db = None
self.init_chess_db()
self.games, self.offs_ply = self.get_records(0)
log.info("%s contains %s game(s)" % (self.path, self.count), extra={"task": "SQL"})
def get_count(self):
""" Number of games in .pgn database """
if self.pgn_is_string:
return len(self.games)
else:
return self.tag_database.count
count = property(get_count)
def get_size(self):
""" Size of .pgn file in bytes """
return os.path.getsize(self.path)
size = property(get_size)
def init_tag_database(self):
""" Create/open .sqlite database of game header tags """
sqlite_path = os.path.splitext(self.path)[0] + '.sqlite'
self.engine = dbmodel.get_engine(sqlite_path)
self.tag_database = TagDatabase(self.engine)
# Import .pgn header tags to .sqlite database
size = self.size
if size > 0 and self.tag_database.count == 0:
if size > 10000000:
drop_indexes(self.engine)
importer = PgnImport(self)
importer.do_import(self.path, progressbar=self.progressbar)
if size > 10000000:
create_indexes(self.engine)
def init_chess_db(self):
""" Create/open polyglot .bin file with extra win/loss/draw stats
using chess_db parser from https://github.com/mcostalba/chess_db
"""
if chess_db_path is not None and self.path and self.size > 0:
try:
if self.progressbar is not None:
self.progressbar.set_text("Creating .bin index file...")
self.chess_db = Parser(engine=(chess_db_path, ))
self.chess_db.open(self.path)
bin_path = os.path.splitext(self.path)[0] + '.bin'
if not os.path.isfile(bin_path):
log.debug("No valid games found in %s" % self.path)
self.chess_db = None
elif getmtime(self.path) > getmtime(bin_path):
self.chess_db.make()
except OSError as err:
self.chess_db = None
log.warning("Failed to sart chess_db parser. OSError %s %s" % (err.errno, err.strerror))
except pexpect.TIMEOUT:
self.chess_db = None
log.warning("chess_db parser failed (pexpect.TIMEOUT)")
except pexpect.EOF:
self.chess_db = None
log.warning("chess_db parser failed (pexpect.EOF)")
def init_scoutfish(self):
""" Create/open .scout database index file to help querying
using scoutfish from https://github.com/mcostalba/scoutfish
"""
if scoutfish_path is not None and self.path and self.size > 0:
try:
if self.progressbar is not None:
self.progressbar.set_text("Creating .scout index file...")
self.scoutfish = Scoutfish(engine=(scoutfish_path, ))
self.scoutfish.open(self.path)
scout_path = os.path.splitext(self.path)[0] + '.scout'
if getmtime(self.path) > getmtime(scout_path):
self.scoutfish.make()
except OSError as err:
self.scoutfish = None
log.warning("Failed to sart scoutfish. OSError %s %s" % (err.errno, err.strerror))
except pexpect.TIMEOUT:
self.scoutfish = None
log.warning("scoutfish failed (pexpect.TIMEOUT)")
except pexpect.EOF:
self.scoutfish = None
log.warning("scoutfish failed (pexpect.EOF)")
def get_book_moves(self, fen):
""" Get move-games-win-loss-draw stat of fen position """
rows = []
if self.chess_db is not None:
move_stat = self.chess_db.find("limit %s skip %s %s" % (1, 0, fen))
for mstat in move_stat["moves"]:
rows.append((mstat["move"], int(mstat["games"]), int(mstat["wins"]), int(mstat["losses"]), int(mstat["draws"])))
return rows
def set_tag_order(self, order_col, is_desc):
self.order_col = order_col
self.is_desc = is_desc
self.tag_database.build_order_by(self.order_col, self.is_desc)
def reset_last_seen(self):
col_max = "ZZZ" if isinstance(self.order_col.type, String) else 2**32
col_min = "" if isinstance(self.order_col.type, String) else -1
if self.is_desc:
self.last_seen = [(col_max, 2**32)]
else:
self.last_seen = [(col_min, -1)]
def set_tag_filter(self, query):
""" Set (now prefixing) text and
create where clause we will use to query header tag .sqlite database
"""
self.tag_query = query
self.tag_database.build_where_tags(self.tag_query)
def set_fen_filter(self, fen):
""" Set fen string we will use to get game offsets from .bin database """
if self.chess_db is not None and fen is not None and fen != FEN_START:
self.fen = fen
else:
self.fen = None
self.tag_database.build_where_offs8(None)
def set_scout_filter(self, query):
""" Set json string we will use to get game offsets from .scout database """
if self.scoutfish is not None and query:
self.scout_query = query
else:
self.scout_query = None
self.tag_database.build_where_offs(None)
self.offs_ply = {}
def get_offs(self, skip, filtered_offs_list=None):
""" Get offsets from .scout database and
create where clause we will use to query header tag .sqlite database
"""
if self.scout_query:
limit = (10000 if self.tag_query else self.limit) + 1
self.scout_query["skip"] = skip
self.scout_query["limit"] = limit
move_stat = self.scoutfish.scout(self.scout_query)
offsets = []
for mstat in move_stat["matches"]:
offs = mstat["ofs"]
if filtered_offs_list is None:
offsets.append(offs)
self.offs_ply[offs] = mstat["ply"][0]
elif offs in filtered_offs_list:
offsets.append(offs)
self.offs_ply[offs] = mstat["ply"][0]
if filtered_offs_list is not None:
# Continue scouting until we get enough good offset if needed
# print(0, move_stat["match count"], len(offsets))
i = 1
while len(offsets) < self.limit and move_stat["match count"] == limit:
self.scout_query["skip"] = i * limit - 1
move_stat = self.scoutfish.scout(self.scout_query)
for mstat in move_stat["matches"]:
offs = mstat["ofs"]
if offs in filtered_offs_list:
offsets.append(offs)
self.offs_ply[offs] = mstat["ply"][0]
# print(i, move_stat["match count"], len(offsets))
i += 1
if len(offsets) > self.limit:
self.tag_database.build_where_offs(offsets[:self.limit])
else:
self.tag_database.build_where_offs(offsets)
def get_offs8(self, skip, filtered_offs_list=None):
""" Get offsets from .bin database and
create where clause we will use to query header tag .sqlite database
"""
if self.fen:
move_stat = self.chess_db.find("limit %s skip %s %s" % (self.limit, skip, self.fen))
offsets = []
for mstat in move_stat["moves"]:
offs = mstat["pgn offsets"]
if filtered_offs_list is None:
offsets += offs
elif offs in filtered_offs_list:
offsets += offs
if len(offsets) > self.limit:
self.tag_database.build_where_offs8(sorted(offsets)[:self.limit])
else:
self.tag_database.build_where_offs8(sorted(offsets))
def get_records(self, direction=FIRST_PAGE):
""" Get game header tag records from .sqlite database in paginated way """
if direction == FIRST_PAGE:
self.skip = 0
self.reset_last_seen()
elif direction == NEXT_PAGE:
if not self.tag_query:
self.skip += self.limit
elif direction == PREV_PAGE:
if len(self.last_seen) == 2:
self.reset_last_seen()
elif len(self.last_seen) > 2:
self.last_seen = self.last_seen[:-2]
if not self.tag_query and self.skip >= self.limit:
self.skip -= self.limit
if self.fen:
self.reset_last_seen()
filtered_offs_list = None
if self.tag_query and (self.fen or self.scout_query):
filtered_offs_list = self.tag_database.get_offsets_for_tags(self.last_seen[-1])
if self.fen:
self.get_offs8(self.skip, filtered_offs_list=filtered_offs_list)
if self.scout_query:
self.get_offs(self.skip, filtered_offs_list=filtered_offs_list)
# No game satisfied scout_query
if self.tag_database.where_offs is None:
return [], {}
records = self.tag_database.get_records(self.last_seen[-1], self.limit)
if records:
self.last_seen.append((records[-1][col2label[self.order_col]], records[-1]["Offset"]))
return records, self.offs_ply
else:
return [], {}
def load_game_tags(self):
""" Reads header tags from pgn if pgn is a one game only StringIO object """
header = collections.defaultdict(str)
header["Id"] = 0
header["Offset"] = 0
for line in self.handle.readlines():
line = line.strip()
if line.startswith('[') and line.endswith(']'):
tag_match = TAG_REGEX.match(line)
if tag_match:
header[tag_match.group(1)] = tag_match.group(2)
else:
break
return header
def loadToModel(self, rec, position=-1, model=None):
""" Parse game text and load game record header tags to a GameModel object """
if not model:
model = GameModel()
if self.pgn_is_string:
rec = self.games[0]
game_date = rec["Date"]
result = rec["Result"]
variant = rec["Variant"]
else:
game_date = self.get_date(rec)
result = reprResult[rec["Result"]]
variant = self.get_variant(rec)
# the seven mandatory PGN headers
model.tags['Event'] = rec["Event"]
model.tags['Site'] = rec["Site"]
model.tags['Date'] = game_date
model.tags['Round'] = rec["Round"]
model.tags['White'] = rec["White"]
model.tags['Black'] = rec["Black"]
model.tags['Result'] = result
if model.tags['Date']:
date_match = re.match(".*(\d{4}).(\d{2}).(\d{2}).*",
model.tags['Date'])
if date_match:
year, month, day = date_match.groups()
model.tags['Year'] = year
model.tags['Month'] = month
model.tags['Day'] = day
# non-mandatory tags
for tag in ('Annotator', 'ECO', 'WhiteElo', 'BlackElo', 'TimeControl'):
value = rec[tag]
if value:
model.tags[tag] = value
else:
model.tags[tag] = ""
if not self.pgn_is_string:
model.info = self.tag_database.get_info(rec)
if model.tags['TimeControl']:
secs, gain = parseTimeControlTag(model.tags['TimeControl'])
model.timed = True
model.timemodel.secs = secs
model.timemodel.gain = gain
model.timemodel.minutes = secs / 60
for tag, color in (('WhiteClock', WHITE), ('BlackClock', BLACK)):
if hasattr(rec, tag):
try:
millisec = parseClockTimeTag(rec[tag])
# We need to fix when FICS reports negative clock time like this
# [TimeControl "180+0"]
# [WhiteClock "0:00:15.867"]
# [BlackClock "23:59:58.820"]
start_sec = (
millisec - 24 * 60 * 60 * 1000
) / 1000. if millisec > 23 * 60 * 60 * 1000 else millisec / 1000.
model.timemodel.intervals[color][0] = start_sec
except ValueError:
raise LoadingError(
"Error parsing '%s'" % tag)
fenstr = rec["FEN"]
if variant:
if variant not in name2variant:
raise LoadingError("Unknown variant %s" % variant)
model.tags["Variant"] = variant
# Fixes for some non statndard Chess960 .pgn
if (fenstr is not None) and variant == "Fischerandom":
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
model.variant = name2variant[variant]
board = LBoard(model.variant.variant)
else:
model.variant = NormalBoard
board = LBoard()
if fenstr:
try:
board.applyFen(fenstr)
except SyntaxError as err:
board.applyFen(FEN_EMPTY)
raise LoadingError(
_("The game can't be loaded, because of an error parsing FEN"),
err.args[0])
else:
board.applyFen(FEN_START)
boards = [board]
del model.moves[:]
del model.variations[:]
self.error = None
movetext = self.get_movetext(rec)
boards = self.parse_movetext(movetext, boards[0], position)
# The parser built a tree of lboard objects, now we have to
# create the high level Board and Move lists...
for board in boards:
if board.lastMove is not None:
model.moves.append(Move(board.lastMove))
self.has_emt = False
self.has_eval = False
def walk(model, node, path):
if node.prev is None:
# initial game board
board = model.variant(setup=node.asFen(), lboard=node)
else:
move = Move(node.lastMove)
try:
board = node.prev.pieceBoard.move(move, lboard=node)
except:
raise LoadingError(
_("Invalid move."),
"%s%s" % (move_count(node, black_periods=True), move))
if node.next is None:
model.variations.append(path + [board])
else:
walk(model, node.next, path + [board])
for child in node.children:
if isinstance(child, list):
if len(child) > 1:
# non empty variation, go walk
walk(model, child[1], list(path))
else:
if not self.has_emt:
self.has_emt = child.find("%emt") >= 0
if not self.has_eval:
self.has_eval = child.find("%eval") >= 0
# Collect all variation paths into a list of board lists
# where the first one will be the boards of mainline game.
# model.boards will allways point to the current shown variation
# which will be model.variations[0] when we are in the mainline.
walk(model, boards[0], [])
model.boards = model.variations[0]
self.has_emt = self.has_emt and "TimeControl" in model.tags
if self.has_emt or self.has_eval:
if self.has_emt:
blacks = len(model.moves) // 2
whites = len(model.moves) - blacks
model.timemodel.intervals = [
[model.timemodel.intervals[0][0]] * (whites + 1),
[model.timemodel.intervals[1][0]] * (blacks + 1),
]
secs, gain = parseTimeControlTag(model.tags['TimeControl'])
model.timemodel.intervals[0][0] = secs
model.timemodel.intervals[1][0] = secs
for ply, board in enumerate(boards):
for child in board.children:
if isinstance(child, str):
if self.has_emt:
match = movetime.search(child)
if match:
movecount, color = divmod(ply + 1, 2)
hour, minute, sec, msec = match.groups()
prev = model.timemodel.intervals[color][
movecount - 1]
hour = 0 if hour is None else int(hour[:-1])
minute = 0 if minute is None else int(minute[:-1])
msec = 0 if msec is None else int(msec)
msec += int(sec) * 1000 + int(
minute) * 60 * 1000 + int(
hour) * 60 * 60 * 1000
model.timemodel.intervals[color][
movecount] = prev - msec / 1000. + gain
if self.has_eval:
match = moveeval.search(child)
if match:
sign, num, fraction, depth = match.groups()
sign = 1 if sign is None or sign == "+" else -1
num = int(num) if int(
num) == MATE_VALUE else int(num)
fraction = 0 if fraction is None else int(
fraction)
value = sign * (num * 100 + fraction)
depth = "" if depth is None else depth
if board.color == BLACK:
value = -value
model.scores[ply] = ("", value, depth)
log.debug("pgn.loadToModel: intervals %s" %
model.timemodel.intervals)
# Find the physical status of the game
model.status, model.reason = getStatus(model.boards[-1])
# Apply result from .pgn if the last position was loaded
if position == -1 or len(model.moves) == position - model.lowply:
status = rec["Result"]
if status in (WHITEWON, BLACKWON) and status != model.status:
model.status = status
model.reason = WON_RESIGN
elif status == DRAW and status != model.status:
model.status = DRAW
model.reason = DRAW_AGREE
# If parsing gave an error we throw it now, to enlarge our possibility
# of being able to continue the game from where it failed.
if self.error:
raise self.error
return model
def parse_movetext(self, string, board, position, variation=False):
"""Recursive parses a movelist part of one game.
Arguments:
srting - str (movelist)
board - lboard (initial position)
position - int (maximum ply to parse)
variation- boolean (True if the string is a variation)"""
boards = []
boards_append = boards.append
last_board = board
if variation:
# this board used only to hold initial variation comments
boards_append(LBoard(board.variant))
else:
# initial game board
boards_append(board)
# status = None
parenthesis = 0
v_string = ""
v_last_board = None
for m in re.finditer(pattern, string):
group, text = m.lastindex, m.group(m.lastindex)
if parenthesis > 0:
v_string += ' ' + text
if group == VARIATION_END:
parenthesis -= 1
if parenthesis == 0:
if last_board.prev is None:
errstr1 = _("Error parsing %(mstr)s") % {"mstr": string}
self.error = LoadingError(errstr1, "")
return boards # , status
v_last_board.children.append(
self.parse_movetext(v_string[:-1], last_board.prev, position, variation=True))
v_string = ""
continue
elif group == VARIATION_START:
parenthesis += 1
if parenthesis == 1:
v_last_board = last_board
if parenthesis == 0:
if group == FULL_MOVE:
if not variation:
if position != -1 and last_board.plyCount >= position:
break
mstr = m.group(MOVE)
try:
lmove = parseSAN(last_board, mstr)
except ParsingError as err:
# TODO: save the rest as comment
# last_board.children.append(string[m.start():])
notation, reason, boardfen = err.args
ply = last_board.plyCount
if ply % 2 == 0:
moveno = "%d." % (ply // 2 + 1)
else:
moveno = "%d..." % (ply // 2 + 1)
errstr1 = _(
"The game can't be read to end, because of an error parsing move %(moveno)s '%(notation)s'.") % {
'moveno': moveno,
'notation': notation}
errstr2 = _("The move failed because %s.") % reason
self.error = LoadingError(errstr1, errstr2)
break
except:
ply = last_board.plyCount
if ply % 2 == 0:
moveno = "%d." % (ply // 2 + 1)
else:
moveno = "%d..." % (ply // 2 + 1)
errstr1 = _(
"Error parsing move %(moveno)s %(mstr)s") % {
"moveno": moveno,
"mstr": mstr}
self.error = LoadingError(errstr1, "")
break
new_board = last_board.clone()
new_board.applyMove(lmove)
if m.group(MOVE_COMMENT):
new_board.nags.append(symbol2nag(m.group(
MOVE_COMMENT)))
new_board.prev = last_board
# set last_board next, except starting a new variation
if variation and last_board == board:
boards[0].next = new_board
else:
last_board.next = new_board
boards_append(new_board)
last_board = new_board
elif group == COMMENT_REST:
last_board.children.append(text[1:])
elif group == COMMENT_BRACE:
comm = text.replace('{\r\n', '{').replace('\r\n}', '}')
comm = comm[1:-1].splitlines()
comment = ' '.join([line.strip() for line in comm])
if variation and last_board == board:
# initial variation comment
boards[0].children.append(comment)
else:
last_board.children.append(comment)
elif group == COMMENT_NAG:
last_board.nags.append(text)
# TODO
elif group == RESULT:
# if text == "1/2":
# status = reprResult.index("1/2-1/2")
# else:
# status = reprResult.index(text)
break
else:
print("Unknown:", text)
return boards # , status
def get_movetext(self, rec):
self.handle.seek(rec["Offset"])
lines = []
line = self.handle.readline()
if not line.strip():
line = self.handle.readline()
while line:
if line.startswith("["):
line = self.handle.readline()
elif line.startswith("%"):
line = self.handle.readline()
elif line.strip():
lines.append(line)
line = self.handle.readline()
elif len(lines) == 0:
line = self.handle.readline()
else:
break
return "".join(lines)
def get_variant(self, rec):
variant = rec["Variant"]
return variants[variant].cecp_name.capitalize() if variant else ""
def get_date(self, rec):
year = rec['Year']
month = rec['Month']
day = rec['Day']
if year and month and day:
tag_date = "%s.%02d.%02d" % (year, month, day)
elif year and month:
tag_date = "%s.%02d" % (year, month)
elif year:
tag_date = "%s" % year
else:
tag_date = ""
return tag_date
class PGNFile(ChessFile):
def __init__(self, handle, progressbar=None):
ChessFile.__init__(self, handle)
self.handle = handle
self.progressbar = progressbar
self.pgn_is_string = isinstance(handle, StringIO)
if self.pgn_is_string:
self.games = [
self.load_game_tags(),
]
else:
self.skip = 0
self.limit = 100
self.order_col = game.c.offset
self.is_desc = False
self.reset_last_seen()
# filter expressions to .sqlite .bin .scout
self.tag_query = None
self.fen = None
self.scout_query = None
self.scoutfish = None
self.chess_db = None
self.sqlite_path = os.path.splitext(self.path)[0] + '.sqlite'
self.engine = dbmodel.get_engine(self.sqlite_path)
self.tag_database = TagDatabase(self.engine)
self.games, self.offs_ply = self.get_records(0)
log.info("%s contains %s game(s)" % (self.path, self.count),
extra={"task": "SQL"})
def get_count(self):
""" Number of games in .pgn database """
if self.pgn_is_string:
return len(self.games)
else:
return self.tag_database.count
count = property(get_count)
def get_size(self):
""" Size of .pgn file in bytes """
return os.path.getsize(self.path)
size = property(get_size)
def close(self):
self.tag_database.close()
ChessFile.close(self)
def init_tag_database(self, importer=None):
""" Create/open .sqlite database of game header tags """
# Import .pgn header tags to .sqlite database
sqlite_path = self.path.replace(".pgn", ".sqlite")
if os.path.isfile(
self.path) and os.path.isfile(sqlite_path) and getmtime(
self.path) > getmtime(sqlite_path):
metadata.drop_all(self.engine)
metadata.create_all(self.engine)
ini_schema_version(self.engine)
size = self.size
if size > 0 and self.tag_database.count == 0:
if size > 10000000:
drop_indexes(self.engine)
if self.progressbar is not None:
from gi.repository import GLib
GLib.idle_add(self.progressbar.set_text,
_("Importing game headers..."))
if importer is None:
importer = PgnImport(self)
importer.initialize()
importer.do_import(self.path, progressbar=self.progressbar)
if size > 10000000 and not importer.cancel:
create_indexes(self.engine)
return importer
def init_chess_db(self):
""" Create/open polyglot .bin file with extra win/loss/draw stats
using chess_db parser from https://github.com/mcostalba/chess_db
"""
if chess_db_path is not None and self.path and self.size > 0:
try:
if self.progressbar is not None:
from gi.repository import GLib
GLib.idle_add(self.progressbar.set_text,
_("Creating .bin index file..."))
self.chess_db = Parser(engine=(chess_db_path, ))
self.chess_db.open(self.path)
bin_path = os.path.splitext(self.path)[0] + '.bin'
if not os.path.isfile(bin_path):
log.debug("No valid games found in %s" % self.path)
self.chess_db = None
elif getmtime(self.path) > getmtime(bin_path):
self.chess_db.make()
except OSError as err:
self.chess_db = None
log.warning("Failed to sart chess_db parser. OSError %s %s" %
(err.errno, err.strerror))
except pexpect.TIMEOUT:
self.chess_db = None
log.warning("chess_db parser failed (pexpect.TIMEOUT)")
except pexpect.EOF:
self.chess_db = None
log.warning("chess_db parser failed (pexpect.EOF)")
def init_scoutfish(self):
""" Create/open .scout database index file to help querying
using scoutfish from https://github.com/mcostalba/scoutfish
"""
if scoutfish_path is not None and self.path and self.size > 0:
try:
if self.progressbar is not None:
from gi.repository import GLib
GLib.idle_add(self.progressbar.set_text,
_("Creating .scout index file..."))
self.scoutfish = Scoutfish(engine=(scoutfish_path, ))
self.scoutfish.open(self.path)
scout_path = os.path.splitext(self.path)[0] + '.scout'
if getmtime(self.path) > getmtime(scout_path):
self.scoutfish.make()
except OSError as err:
self.scoutfish = None
log.warning("Failed to sart scoutfish. OSError %s %s" %
(err.errno, err.strerror))
except pexpect.TIMEOUT:
self.scoutfish = None
log.warning("scoutfish failed (pexpect.TIMEOUT)")
except pexpect.EOF:
self.scoutfish = None
log.warning("scoutfish failed (pexpect.EOF)")
def get_book_moves(self, fen):
""" Get move-games-win-loss-draw stat of fen position """
rows = []
if self.chess_db is not None:
move_stat = self.chess_db.find("limit %s skip %s %s" % (1, 0, fen))
for mstat in move_stat["moves"]:
rows.append(
(mstat["move"], int(mstat["games"]), int(mstat["wins"]),
int(mstat["losses"]), int(mstat["draws"])))
return rows
def has_position(self, fen):
# ChessDB (prioritary)
if self.chess_db is not None:
ret = self.chess_db.find("limit %s skip %s %s" % (1, 0, fen))
if len(ret["moves"]) > 0:
return TOOL_CHESSDB, True
# Scoutfish (alternate by approximation)
if self.scoutfish is not None:
q = {"limit": 1, "skip": 0, "sub-fen": fen}
ret = self.scoutfish.scout(q)
if ret["match count"] > 0:
return TOOL_SCOUTFISH, True
return TOOL_NONE, False
def set_tag_order(self, order_col, is_desc):
self.order_col = order_col
self.is_desc = is_desc
self.tag_database.build_order_by(self.order_col, self.is_desc)
def reset_last_seen(self):
col_max = "ZZZ" if isinstance(self.order_col.type, String) else 2**32
col_min = "" if isinstance(self.order_col.type, String) else -1
if self.is_desc:
self.last_seen = [(col_max, 2**32)]
else:
self.last_seen = [(col_min, -1)]
def set_tag_filter(self, query):
""" Set (now prefixing) text and
create where clause we will use to query header tag .sqlite database
"""
self.tag_query = query
self.tag_database.build_where_tags(self.tag_query)
def set_fen_filter(self, fen):
""" Set fen string we will use to get game offsets from .bin database """
if self.chess_db is not None and fen is not None and fen != FEN_START:
self.fen = fen
else:
self.fen = None
self.tag_database.build_where_offs8(None)
def set_scout_filter(self, query):
""" Set json string we will use to get game offsets from .scout database """
if self.scoutfish is not None and query:
self.scout_query = query
else:
self.scout_query = None
self.tag_database.build_where_offs(None)
self.offs_ply = {}
def get_offs(self, skip, filtered_offs_list=None):
""" Get offsets from .scout database and
create where clause we will use to query header tag .sqlite database
"""
if self.scout_query:
limit = (10000 if self.tag_query else self.limit) + 1
self.scout_query["skip"] = skip
self.scout_query["limit"] = limit
move_stat = self.scoutfish.scout(self.scout_query)
offsets = []
for mstat in move_stat["matches"]:
offs = mstat["ofs"]
if filtered_offs_list is None:
offsets.append(offs)
self.offs_ply[offs] = mstat["ply"][0]
elif offs in filtered_offs_list:
offsets.append(offs)
self.offs_ply[offs] = mstat["ply"][0]
if filtered_offs_list is not None:
# Continue scouting until we get enough good offset if needed
# print(0, move_stat["match count"], len(offsets))
i = 1
while len(offsets
) < self.limit and move_stat["match count"] == limit:
self.scout_query["skip"] = i * limit - 1
move_stat = self.scoutfish.scout(self.scout_query)
for mstat in move_stat["matches"]:
offs = mstat["ofs"]
if offs in filtered_offs_list:
offsets.append(offs)
self.offs_ply[offs] = mstat["ply"][0]
# print(i, move_stat["match count"], len(offsets))
i += 1
if len(offsets) > self.limit:
self.tag_database.build_where_offs(offsets[:self.limit])
else:
self.tag_database.build_where_offs(offsets)
def get_offs8(self, skip, filtered_offs_list=None):
""" Get offsets from .bin database and
create where clause we will use to query header tag .sqlite database
"""
if self.fen:
move_stat = self.chess_db.find("limit %s skip %s %s" %
(self.limit, skip, self.fen))
offsets = []
for mstat in move_stat["moves"]:
offs = mstat["pgn offsets"]
if filtered_offs_list is None:
offsets += offs
elif offs in filtered_offs_list:
offsets += offs
if len(offsets) > self.limit:
self.tag_database.build_where_offs8(
sorted(offsets)[:self.limit])
else:
self.tag_database.build_where_offs8(sorted(offsets))
def get_records(self, direction=FIRST_PAGE):
""" Get game header tag records from .sqlite database in paginated way """
if direction == FIRST_PAGE:
self.skip = 0
self.reset_last_seen()
elif direction == NEXT_PAGE:
if not self.tag_query:
self.skip += self.limit
elif direction == PREV_PAGE:
if len(self.last_seen) == 2:
self.reset_last_seen()
elif len(self.last_seen) > 2:
self.last_seen = self.last_seen[:-2]
if not self.tag_query and self.skip >= self.limit:
self.skip -= self.limit
if self.fen:
self.reset_last_seen()
filtered_offs_list = None
if self.tag_query and (self.fen or self.scout_query):
filtered_offs_list = self.tag_database.get_offsets_for_tags(
self.last_seen[-1])
if self.fen:
self.get_offs8(self.skip, filtered_offs_list=filtered_offs_list)
if self.scout_query:
self.get_offs(self.skip, filtered_offs_list=filtered_offs_list)
# No game satisfied scout_query
if self.tag_database.where_offs is None:
return [], {}
records = self.tag_database.get_records(self.last_seen[-1], self.limit)
if records:
self.last_seen.append((records[-1][col2label[self.order_col]],
records[-1]["Offset"]))
return records, self.offs_ply
else:
return [], {}
def load_game_tags(self):
""" Reads header tags from pgn if pgn is a one game only StringIO object """
header = collections.defaultdict(str)
header["Id"] = 0
header["Offset"] = 0
for line in self.handle.readlines():
line = line.strip()
if line.startswith('[') and line.endswith(']'):
tag_match = TAG_REGEX.match(line)
if tag_match:
value = tag_match.group(2)
value = value.replace("\\\"", "\"")
value = value.replace("\\\\", "\\")
header[tag_match.group(1)] = value
else:
break
return header
def loadToModel(self, rec, position=-1, model=None):
""" Parse game text and load game record header tags to a GameModel object """
if model is None:
model = GameModel()
if self.pgn_is_string:
rec = self.games[0]
# Load mandatory tags
for tag in mandatory_tags:
model.tags[tag] = rec[tag]
# Load other tags
for tag in ('WhiteElo', 'BlackElo', 'ECO', 'TimeControl', 'Annotator'):
model.tags[tag] = rec[tag]
if self.pgn_is_string:
for tag in rec:
if isinstance(rec[tag], str) and rec[tag]:
model.tags[tag] = rec[tag]
else:
model.info = self.tag_database.get_info(rec)
extra_tags = self.tag_database.get_exta_tags(rec)
for et in extra_tags:
model.tags[et['tag_name']] = et['tag_value']
if self.pgn_is_string:
variant = rec["Variant"].capitalize()
else:
variant = self.get_variant(rec)
if model.tags['TimeControl']:
tc = parseTimeControlTag(model.tags['TimeControl'])
if tc is not None:
secs, gain, moves = tc
model.timed = True
model.timemodel.secs = secs
model.timemodel.gain = gain
model.timemodel.minutes = secs / 60
model.timemodel.moves = moves
for tag, color in (('WhiteClock', WHITE), ('BlackClock',
BLACK)):
if tag in model.tags:
try:
millisec = parseClockTimeTag(model.tags[tag])
# We need to fix when FICS reports negative clock time like this
# [TimeControl "180+0"]
# [WhiteClock "0:00:15.867"]
# [BlackClock "23:59:58.820"]
start_sec = (
millisec - 24 * 60 * 60 * 1000
) / 1000. if millisec > 23 * 60 * 60 * 1000 else millisec / 1000.
model.timemodel.intervals[color][0] = start_sec
except ValueError:
raise LoadingError("Error parsing '%s'" % tag)
fenstr = rec["FEN"]
if variant:
if variant not in name2variant:
raise LoadingError("Unknown variant %s" % variant)
model.tags["Variant"] = variant
# Fixes for some non statndard Chess960 .pgn
if (fenstr is not None) and variant == "Fischerandom":
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
model.variant = name2variant[variant]
board = LBoard(model.variant.variant)
else:
model.variant = NormalBoard
board = LBoard()
if fenstr:
try:
board.applyFen(fenstr)
model.tags["FEN"] = fenstr
except SyntaxError as err:
board.applyFen(FEN_EMPTY)
raise LoadingError(
_("The game can't be loaded, because of an error parsing FEN"
), err.args[0])
else:
board.applyFen(FEN_START)
boards = [board]
del model.moves[:]
del model.variations[:]
self.error = None
movetext = self.get_movetext(rec)
boards = self.parse_movetext(movetext, boards[0], position)
# The parser built a tree of lboard objects, now we have to
# create the high level Board and Move lists...
for board in boards:
if board.lastMove is not None:
model.moves.append(Move(board.lastMove))
self.has_emt = False
self.has_eval = False
def _create_board(model, node):
if node.prev is None:
# initial game board
board = model.variant(setup=node.asFen(), lboard=node)
else:
move = Move(node.lastMove)
try:
board = node.prev.pieceBoard.move(move, lboard=node)
except Exception:
raise LoadingError(
_("Invalid move."),
"%s%s" % (move_count(node, black_periods=True), move))
return board
def walk(model, node, path):
boards = path
stack = []
current = node
while current is not None:
board = _create_board(model, current)
boards.append(board)
stack.append(current)
current = current.next
else:
model.variations.append(list(boards))
while stack:
current = stack.pop()
boards.pop()
for child in current.children:
if isinstance(child, list):
if len(child) > 1:
# non empty variation, go walk
walk(model, child[1], list(boards))
else:
if not self.has_emt:
self.has_emt = child.find("%emt") >= 0
if not self.has_eval:
self.has_eval = child.find("%eval") >= 0
# Collect all variation paths into a list of board lists
# where the first one will be the boards of mainline game.
# model.boards will allways point to the current shown variation
# which will be model.variations[0] when we are in the mainline.
walk(model, boards[0], [])
model.boards = model.variations[0]
self.has_emt = self.has_emt and model.timed
if self.has_emt or self.has_eval:
if self.has_emt:
blacks = len(model.moves) // 2
whites = len(model.moves) - blacks
model.timemodel.intervals = [
[model.timemodel.intervals[0][0]] * (whites + 1),
[model.timemodel.intervals[1][0]] * (blacks + 1),
]
model.timemodel.intervals[0][0] = secs
model.timemodel.intervals[1][0] = secs
for ply, board in enumerate(boards):
for child in board.children:
if isinstance(child, str):
if self.has_emt:
match = move_time_re.search(child)
if match:
movecount, color = divmod(ply + 1, 2)
hour, minute, sec, msec = match.groups()
prev = model.timemodel.intervals[color][
movecount - 1]
hour = 0 if hour is None else int(hour[:-1])
minute = 0 if minute is None else int(
minute[:-1])
msec = 0 if msec is None else int(msec)
msec += int(sec) * 1000 + int(
minute) * 60 * 1000 + int(
hour) * 60 * 60 * 1000
model.timemodel.intervals[color][
movecount] = prev - msec / 1000. + gain
if self.has_eval:
match = move_eval_re.search(child)
if match:
sign, num, fraction, depth = match.groups()
sign = 1 if sign is None or sign == "+" else -1
num = int(num)
fraction = 0 if fraction is None else int(
fraction)
value = sign * (num * 100 + fraction)
depth = "" if depth is None else depth
if board.color == BLACK:
value = -value
model.scores[ply] = ("", value, depth)
log.debug("pgn.loadToModel: intervals %s" %
model.timemodel.intervals)
# Find the physical status of the game
model.status, model.reason = getStatus(model.boards[-1])
# Apply result from .pgn if the last position was loaded
if position == -1 or len(model.moves) == position - model.lowply:
if self.pgn_is_string:
result = rec["Result"]
if result in pgn2Const:
status = pgn2Const[result]
else:
status = RUNNING
else:
status = rec["Result"]
if status in (WHITEWON, BLACKWON) and status != model.status:
model.status = status
model.reason = WON_RESIGN
elif status == DRAW and status != model.status:
model.status = DRAW
model.reason = DRAW_AGREE
if model.timed:
model.timemodel.movingColor = model.boards[-1].color
# If parsing gave an error we throw it now, to enlarge our possibility
# of being able to continue the game from where it failed.
if self.error:
raise self.error
return model
def parse_movetext(self, string, board, position, variation=False):
"""Recursive parses a movelist part of one game.
Arguments:
srting - str (movelist)
board - lboard (initial position)
position - int (maximum ply to parse)
variation- boolean (True if the string is a variation)"""
boards = []
boards_append = boards.append
last_board = board
if variation:
# this board used only to hold initial variation comments
boards_append(LBoard(board.variant))
else:
# initial game board
boards_append(board)
# status = None
parenthesis = 0
v_string = ""
v_last_board = None
for m in re.finditer(pattern, string):
group, text = m.lastindex, m.group(m.lastindex)
if parenthesis > 0:
v_string += ' ' + text
if group == VARIATION_END:
parenthesis -= 1
if parenthesis == 0:
if last_board.prev is None:
errstr1 = _("Error parsing %(mstr)s") % {
"mstr": string
}
self.error = LoadingError(errstr1, "")
return boards # , status
v_last_board.children.append(
self.parse_movetext(v_string[:-1],
last_board.prev,
position,
variation=True))
v_string = ""
continue
elif group == VARIATION_START:
parenthesis += 1
if parenthesis == 1:
v_last_board = last_board
if parenthesis == 0:
if group == FULL_MOVE:
if not variation:
if position != -1 and last_board.plyCount >= position:
break
mstr = m.group(MOVE)
try:
lmove = parseAny(last_board, mstr)
except ParsingError as err:
# TODO: save the rest as comment
# last_board.children.append(string[m.start():])
notation, reason, boardfen = err.args
ply = last_board.plyCount
if ply % 2 == 0:
moveno = "%d." % (ply // 2 + 1)
else:
moveno = "%d..." % (ply // 2 + 1)
errstr1 = _(
"The game can't be read to end, because of an error parsing move %(moveno)s '%(notation)s'."
) % {
'moveno': moveno,
'notation': notation
}
errstr2 = _("The move failed because %s.") % reason
self.error = LoadingError(errstr1, errstr2)
break
except Exception:
ply = last_board.plyCount
if ply % 2 == 0:
moveno = "%d." % (ply // 2 + 1)
else:
moveno = "%d..." % (ply // 2 + 1)
errstr1 = _(
"Error parsing move %(moveno)s %(mstr)s") % {
"moveno": moveno,
"mstr": mstr
}
self.error = LoadingError(errstr1, "")
break
new_board = last_board.clone()
new_board.applyMove(lmove)
if m.group(MOVE_COMMENT):
new_board.nags.append(symbol2nag(
m.group(MOVE_COMMENT)))
new_board.prev = last_board
# set last_board next, except starting a new variation
if variation and last_board == board:
boards[0].next = new_board
else:
last_board.next = new_board
boards_append(new_board)
last_board = new_board
elif group == COMMENT_REST:
last_board.children.append(text[1:])
elif group == COMMENT_BRACE:
comm = text.replace('{\r\n', '{').replace('\r\n}', '}')
# Preserve new lines of lichess study comments
if self.path is not None and "lichess_study_" in self.path:
comment = comm[1:-1]
else:
comm = comm[1:-1].splitlines()
comment = ' '.join([line.strip() for line in comm])
if variation and last_board == board:
# initial variation comment
boards[0].children.append(comment)
else:
last_board.children.append(comment)
elif group == COMMENT_NAG:
last_board.nags.append(text)
# TODO
elif group == RESULT:
# if text == "1/2":
# status = reprResult.index("1/2-1/2")
# else:
# status = reprResult.index(text)
break
else:
print("Unknown:", text)
return boards # , status
def get_movetext(self, rec):
self.handle.seek(rec["Offset"])
in_comment = False
lines = []
line = self.handle.readline()
if not line.strip():
line = self.handle.readline()
while line:
# escape non-PGN data line
if line.startswith("%"):
line = self.handle.readline()
continue
# header tag line
if not in_comment and line.startswith("["):
line = self.handle.readline()
continue
# update in_comment state
if (not in_comment and "{" in line) or (in_comment
and "}" in line):
in_comment = line.rfind("{") > line.rfind("}")
# if there is something add it
if line.strip():
if not self.pgn_is_string and self.handle.pgn_encoding != PGN_ENCODING:
line = line.encode(PGN_ENCODING).decode(
self.handle.pgn_encoding)
lines.append(line)
line = self.handle.readline()
# if line is empty it should be the game separator line except...
elif len(lines) == 0 or in_comment:
if in_comment:
lines.append(line)
line = self.handle.readline()
else:
break
return "".join(lines)
def get_variant(self, rec):
variant = rec["Variant"]
return variants[variant].cecp_name.capitalize() if variant else ""
class PGNFile(ChessFile):
def __init__(self, handle, progressbar):
ChessFile.__init__(self, handle)
self.handle = handle
self.progressbar = progressbar
self.pgn_is_string = isinstance(handle, StringIO)
if self.pgn_is_string:
self.games = [
self.load_game_tags(),
]
else:
self.skip = 0
self.limit = 100
self.last_seen_offs = [-1]
# filter expressions to .sqlite .bin .scout
self.text = ""
self.fen = ""
self.query = {}
self.init_tag_database()
self.scoutfish = None
self.init_scoutfish()
self.chess_db = None
self.init_chess_db()
self.games, self.offs_ply = self.get_records(0)
log.info("%s contains %s game(s)" % (self.path, self.count),
extra={"task": "SQL"})
def get_count(self):
if self.pgn_is_string:
return len(self.games)
else:
return self.tag_database.count
count = property(get_count)
def get_size(self):
return os.path.getsize(self.path)
size = property(get_size)
def init_tag_database(self):
sqlite_path = os.path.splitext(self.path)[0] + '.sqlite'
self.engine = dbmodel.get_engine(sqlite_path)
self.tag_database = TagDatabase(self.engine)
# Import .pgn header tags to .sqlite database
size = self.size
if size > 0 and self.tag_database.count == 0:
if size > 10000000:
drop_indexes(self.engine)
importer = PgnImport(self)
importer.do_import(self.path, progressbar=self.progressbar)
if size > 10000000:
create_indexes(self.engine)
def init_chess_db(self):
# Create/open polyglot .bin file with extra win/loss/draw stats
# using chess_db parser from https://github.com/mcostalba/chess_db
if chess_db_path is not None and self.path and self.size > 0:
try:
if self.progressbar is not None:
self.progressbar.set_text("Creating .bin index file...")
self.chess_db = Parser(engine=chess_db_path)
self.chess_db.open(self.path)
bin_path = os.path.splitext(self.path)[0] + '.bin'
if not os.path.isfile(bin_path):
log.debug("No valid games found in %s" % self.path)
self.chess_db = None
elif getmtime(self.path) > getmtime(bin_path):
self.chess_db.make()
except OSError as err:
self.chess_db = None
log.debug("Failed to sart chess_db parser. OSError %s %s" %
(err.errno, err.strerror))
except pexpect.TIMEOUT:
self.chess_db = None
print("chess_db parser failed (pexpect.TIMEOUT)")
except pexpect.EOF:
self.chess_db = None
print("chess_db parser failed (pexpect.EOF)")
def init_scoutfish(self):
# Create/open .scout database index file to help querying
# using scoutfish from https://github.com/mcostalba/scoutfish
if scoutfish_path is not None and self.path and self.size > 0:
try:
if self.progressbar is not None:
self.progressbar.set_text("Creating .scout index file...")
self.scoutfish = Scoutfish(engine=scoutfish_path)
self.scoutfish.open(self.path)
scout_path = os.path.splitext(self.path)[0] + '.scout'
if getmtime(self.path) > getmtime(scout_path):
self.scoutfish.make()
except OSError as err:
self.scoutfish = None
log.debug("Failed to sart scoutfish. OSError %s %s" %
(err.errno, err.strerror))
except pexpect.TIMEOUT:
self.scoutfish = None
print("scoutfish failed (pexpect.TIMEOUT)")
except pexpect.EOF:
self.scoutfish = None
print("scoutfish failed (pexpect.EOF)")
def get_book_moves(self, fen):
rows = []
if self.chess_db is not None:
move_stat = self.chess_db.find("limit %s skip %s %s" % (1, 0, fen))
for stat in move_stat["moves"]:
rows.append(
(stat["move"], int(stat["games"]), int(stat["wins"]),
int(stat["losses"]), int(stat["draws"])))
return rows
def set_tags_filter(self, text):
self.text = text
self.tag_database.build_where_tags(text)
def set_fen_filter(self, fen):
if self.chess_db is not None and fen != FEN_START:
self.fen = fen
else:
self.fen = ""
def set_scout_filter(self, query):
if self.scoutfish is not None and query:
self.query = query
else:
self.query = {}
def get_offs(self, off, skip):
if self.query:
self.query["skip"] = skip
self.query["limit"] = self.limit + 1
move_stat = self.scoutfish.scout(self.query)
offsets = []
for stat in move_stat["matches"]:
offsets.append(stat["ofs"])
self.offs_ply[stat["ofs"]] = stat["ply"][0]
off = sorted(off + offsets)[:self.limit]
self.tag_database.build_where_offs(off)
self.has_more_where_offs = len(offsets) == self.limit + 1
else:
self.tag_database.build_where_offs(None)
self.has_more_where_offs = False
self.offs_ply = {}
return off
def get_offs8(self, off8, skip):
# TODO: how pagination of offsets from .sqlite and .bin and .csout will work together?
# "find" gives offsets in random order because
# entries in .bin are stored in polyglot key order while
# entries in .scout are stored in offset order
if self.fen:
move_stat = self.chess_db.find("limit %s skip %s %s" %
(self.limit + 1, skip, self.fen))
offsets = []
for stat in move_stat["moves"]:
offsets += stat["pgn offsets"]
off8 = sorted(off8 + offsets)[:self.limit]
self.tag_database.build_where_offs8(off8)
self.has_more_where_offs8 = len(offsets) == self.limit + 1
else:
self.tag_database.build_where_offs8(None)
self.has_more_where_offs8 = False
return off8
def get_records(self, direction=FIRST_PAGE):
if direction == FIRST_PAGE:
self.skip = 0
self.last_seen_offs = [-1]
elif direction == NEXT_PAGE:
if not self.text:
self.skip += self.limit
elif direction == PREV_PAGE:
if len(self.last_seen_offs) == 2:
self.last_seen_offs = [-1]
elif len(self.last_seen_offs) > 2:
self.last_seen_offs = self.last_seen_offs[:-2]
if not self.text and self.skip >= self.limit:
self.skip -= self.limit
off = self.get_offs([], self.skip)
off8 = self.get_offs8([], self.skip)
if self.fen:
self.last_seen_offs = [-1]
records = self.tag_database.get_records(self.last_seen_offs[-1],
self.limit)
count_records = len(records)
if count_records < self.limit and direction in (FIRST_PAGE, NEXT_PAGE):
if self.text:
while count_records < self.limit and self.has_more_where_offs:
self.skip += self.limit
off = self.get_offs(off, self.skip)
records = self.tag_database.get_records(
self.last_seen_offs[-1], self.limit)
count_records = len(records)
else:
if self.fen and self.has_more_where_offs8:
off8 = []
self.get_offs8(off8, self.skip)
records = self.tag_database.get_records(
self.last_seen_offs[-1], self.limit)
elif self.query and self.has_more_where_offs:
off = []
self.get_offs(off, self.skip)
records = self.tag_database.get_records(
self.last_seen_offs[-1], self.limit)
if records:
self.last_seen_offs.append(records[-1]["Offset"])
return records, self.offs_ply
else:
return [], {}
def load_game_tags(self):
""" Reads header tags from pgn if pgn is a one game only StringIO object """
header = collections.defaultdict(str)
header["Id"] = 0
header["Offset"] = 0
for line in self.handle.readlines():
line = line.strip()
if line.startswith('[') and line.endswith(']'):
tag_match = TAG_REGEX.match(line)
if tag_match:
header[tag_match.group(1)] = tag_match.group(2)
else:
break
return header
def loadToModel(self, rec, position=-1, model=None):
""" Parse game text and load game record header tags to a GameModel object """
if not model:
model = GameModel()
if self.pgn_is_string:
rec = self.games[0]
game_date = rec["Date"]
result = rec["Result"]
variant = rec["Variant"]
else:
game_date = self.get_date(rec)
result = reprResult[rec["Result"]]
variant = self.get_variant(rec)
# the seven mandatory PGN headers
model.tags['Event'] = rec["Event"]
model.tags['Site'] = rec["Site"]
model.tags['Date'] = game_date
model.tags['Round'] = rec["Round"]
model.tags['White'] = rec["White"]
model.tags['Black'] = rec["Black"]
model.tags['Result'] = result
if model.tags['Date']:
date_match = re.match(".*(\d{4}).(\d{2}).(\d{2}).*",
model.tags['Date'])
if date_match:
year, month, day = date_match.groups()
model.tags['Year'] = year
model.tags['Month'] = month
model.tags['Day'] = day
# non-mandatory tags
for tag in ('Annotator', 'ECO', 'WhiteElo', 'BlackElo', 'TimeControl'):
value = rec[tag]
if value:
model.tags[tag] = value
else:
model.tags[tag] = ""
if not self.pgn_is_string:
model.info = self.tag_database.get_info(rec)
if model.tags['TimeControl']:
secs, gain = parseTimeControlTag(model.tags['TimeControl'])
model.timed = True
model.timemodel.secs = secs
model.timemodel.gain = gain
model.timemodel.minutes = secs / 60
for tag, color in (('WhiteClock', WHITE), ('BlackClock', BLACK)):
if hasattr(rec, tag):
try:
millisec = parseClockTimeTag(rec[tag])
# We need to fix when FICS reports negative clock time like this
# [TimeControl "180+0"]
# [WhiteClock "0:00:15.867"]
# [BlackClock "23:59:58.820"]
start_sec = (
millisec - 24 * 60 * 60 * 1000
) / 1000. if millisec > 23 * 60 * 60 * 1000 else millisec / 1000.
model.timemodel.intervals[color][0] = start_sec
except ValueError:
raise LoadingError("Error parsing '%s'" % tag)
fenstr = rec["FEN"]
if variant:
if variant not in name2variant:
raise LoadingError("Unknown variant %s" % variant)
model.tags["Variant"] = variant
# Fixes for some non statndard Chess960 .pgn
if (fenstr is not None) and variant == "Fischerandom":
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
model.variant = name2variant[variant]
board = LBoard(model.variant.variant)
else:
model.variant = NormalBoard
board = LBoard()
if fenstr:
try:
board.applyFen(fenstr)
except SyntaxError as err:
board.applyFen(FEN_EMPTY)
raise LoadingError(
_("The game can't be loaded, because of an error parsing FEN"
), err.args[0])
else:
board.applyFen(FEN_START)
boards = [board]
del model.moves[:]
del model.variations[:]
self.error = None
movetext = self.get_movetext(rec)
boards = self.parse_movetext(movetext, boards[0], position)
# The parser built a tree of lboard objects, now we have to
# create the high level Board and Move lists...
for board in boards:
if board.lastMove is not None:
model.moves.append(Move(board.lastMove))
self.has_emt = False
self.has_eval = False
def walk(model, node, path):
if node.prev is None:
# initial game board
board = model.variant(setup=node.asFen(), lboard=node)
else:
move = Move(node.lastMove)
try:
board = node.prev.pieceBoard.move(move, lboard=node)
except:
raise LoadingError(
_("Invalid move."),
"%s%s" % (move_count(node, black_periods=True), move))
if node.next is None:
model.variations.append(path + [board])
else:
walk(model, node.next, path + [board])
for child in node.children:
if isinstance(child, list):
if len(child) > 1:
# non empty variation, go walk
walk(model, child[1], list(path))
else:
if not self.has_emt:
self.has_emt = child.find("%emt") >= 0
if not self.has_eval:
self.has_eval = child.find("%eval") >= 0
# Collect all variation paths into a list of board lists
# where the first one will be the boards of mainline game.
# model.boards will allways point to the current shown variation
# which will be model.variations[0] when we are in the mainline.
walk(model, boards[0], [])
model.boards = model.variations[0]
self.has_emt = self.has_emt and "TimeControl" in model.tags
if self.has_emt or self.has_eval:
if self.has_emt:
blacks = len(model.moves) // 2
whites = len(model.moves) - blacks
model.timemodel.intervals = [
[model.timemodel.intervals[0][0]] * (whites + 1),
[model.timemodel.intervals[1][0]] * (blacks + 1),
]
secs, gain = parseTimeControlTag(model.tags['TimeControl'])
model.timemodel.intervals[0][0] = secs
model.timemodel.intervals[1][0] = secs
for ply, board in enumerate(boards):
for child in board.children:
if isinstance(child, basestring):
if self.has_emt:
match = movetime.search(child)
if match:
movecount, color = divmod(ply + 1, 2)
hour, minute, sec, msec = match.groups()
prev = model.timemodel.intervals[color][
movecount - 1]
hour = 0 if hour is None else int(hour[:-1])
minute = 0 if minute is None else int(
minute[:-1])
msec = 0 if msec is None else int(msec)
msec += int(sec) * 1000 + int(
minute) * 60 * 1000 + int(
hour) * 60 * 60 * 1000
model.timemodel.intervals[color][
movecount] = prev - msec / 1000. + gain
if self.has_eval:
match = moveeval.search(child)
if match:
sign, num, fraction, depth = match.groups()
sign = 1 if sign is None or sign == "+" else -1
num = int(num) if int(
num) == MATE_VALUE else int(num)
fraction = 0 if fraction is None else int(
fraction)
value = sign * (num * 100 + fraction)
depth = "" if depth is None else depth
if board.color == BLACK:
value = -value
model.scores[ply] = ("", value, depth)
log.debug("pgn.loadToModel: intervals %s" %
model.timemodel.intervals)
# Find the physical status of the game
model.status, model.reason = getStatus(model.boards[-1])
# Apply result from .pgn if the last position was loaded
if position == -1 or len(model.moves) == position - model.lowply:
status = rec["Result"]
if status in (WHITEWON, BLACKWON) and status != model.status:
model.status = status
model.reason = WON_RESIGN
elif status == DRAW and status != model.status:
model.status = DRAW
model.reason = DRAW_AGREE
# If parsing gave an error we throw it now, to enlarge our possibility
# of being able to continue the game from where it failed.
if self.error:
raise self.error
return model
def parse_movetext(self, string, board, position, variation=False):
"""Recursive parses a movelist part of one game.
Arguments:
srting - str (movelist)
board - lboard (initial position)
position - int (maximum ply to parse)
variation- boolean (True if the string is a variation)"""
boards = []
boards_append = boards.append
last_board = board
if variation:
# this board used only to hold initial variation comments
boards_append(LBoard(board.variant))
else:
# initial game board
boards_append(board)
# status = None
parenthesis = 0
v_string = ""
v_last_board = None
for m in re.finditer(pattern, string):
group, text = m.lastindex, m.group(m.lastindex)
if parenthesis > 0:
v_string += ' ' + text
if group == VARIATION_END:
parenthesis -= 1
if parenthesis == 0:
if last_board.prev is None:
errstr1 = _("Error parsing %(mstr)s") % {
"mstr": string
}
self.error = LoadingError(errstr1, "")
return boards # , status
v_last_board.children.append(
self.parse_movetext(v_string[:-1],
last_board.prev,
position,
variation=True))
v_string = ""
continue
elif group == VARIATION_START:
parenthesis += 1
if parenthesis == 1:
v_last_board = last_board
if parenthesis == 0:
if group == FULL_MOVE:
if not variation:
if position != -1 and last_board.plyCount >= position:
break
mstr = m.group(MOVE)
try:
lmove = parseSAN(last_board, mstr)
except ParsingError as err:
# TODO: save the rest as comment
# last_board.children.append(string[m.start():])
notation, reason, boardfen = err.args
ply = last_board.plyCount
if ply % 2 == 0:
moveno = "%d." % (ply // 2 + 1)
else:
moveno = "%d..." % (ply // 2 + 1)
errstr1 = _(
"The game can't be read to end, because of an error parsing move %(moveno)s '%(notation)s'."
) % {
'moveno': moveno,
'notation': notation
}
errstr2 = _("The move failed because %s.") % reason
self.error = LoadingError(errstr1, errstr2)
break
except:
ply = last_board.plyCount
if ply % 2 == 0:
moveno = "%d." % (ply // 2 + 1)
else:
moveno = "%d..." % (ply // 2 + 1)
errstr1 = _(
"Error parsing move %(moveno)s %(mstr)s") % {
"moveno": moveno,
"mstr": mstr
}
self.error = LoadingError(errstr1, "")
break
new_board = last_board.clone()
new_board.applyMove(lmove)
if m.group(MOVE_COMMENT):
new_board.nags.append(symbol2nag(
m.group(MOVE_COMMENT)))
new_board.prev = last_board
# set last_board next, except starting a new variation
if variation and last_board == board:
boards[0].next = new_board
else:
last_board.next = new_board
boards_append(new_board)
last_board = new_board
elif group == COMMENT_REST:
last_board.children.append(text[1:])
elif group == COMMENT_BRACE:
comm = text.replace('{\r\n', '{').replace('\r\n}', '}')
comm = comm[1:-1].splitlines()
comment = ' '.join([line.strip() for line in comm])
if variation and last_board == board:
# initial variation comment
boards[0].children.append(comment)
else:
last_board.children.append(comment)
elif group == COMMENT_NAG:
last_board.nags.append(text)
# TODO
elif group == RESULT:
# if text == "1/2":
# status = reprResult.index("1/2-1/2")
# else:
# status = reprResult.index(text)
break
else:
print("Unknown:", text)
return boards # , status
def get_movetext(self, rec):
self.handle.seek(rec["Offset"])
lines = []
line = self.handle.readline()
if not line.strip():
line = self.handle.readline()
while line:
if line.startswith("["):
line = self.handle.readline()
elif line.startswith("%"):
line = self.handle.readline()
elif line.strip():
lines.append(line)
line = self.handle.readline()
elif len(lines) == 0:
line = self.handle.readline()
else:
break
return "".join(lines)
def get_variant(self, rec):
variant = rec["Variant"]
return variants[variant].cecp_name.capitalize() if variant else ""
def get_date(self, rec):
year = rec['Year']
month = rec['Month']
day = rec['Day']
if year and month and day:
tag_date = "%s.%02d.%02d" % (year, month, day)
elif year and month:
tag_date = "%s.%02d" % (year, month)
elif year:
tag_date = "%s" % year
else:
tag_date = ""
return tag_date