def start_puzzle_from(filename, index=None):
if filename.lower().endswith(".pgn"):
if filename.startswith("lichess_study"):
chessfile = PGNFile(protoopen(addDataPrefix("learn/puzzles/%s" % filename), encoding="utf-8"))
else:
chessfile = PGNFile(protoopen(addDataPrefix("learn/puzzles/%s" % filename)))
chessfile.limit = 1000
chessfile.init_tag_database()
elif filename.lower().endswith(".olv"):
chessfile = OLVFile(protoopen(addDataPrefix("learn/puzzles/%s" % filename), encoding="utf-8"))
records, plys = chessfile.get_records()
progress = puzzles_solving_progress.get(filename, [0] * chessfile.count)
if index is None:
index = progress.index(0)
rec = records[index]
timemodel = TimeModel(0, 0)
gamemodel = LearnModel(timemodel)
chessfile.loadToModel(rec, 0, gamemodel)
start_puzzle_game(gamemodel, filename, records, index, rec)
def setUp(self):
self.f1 = protoopen('gamefiles/3fold.pgn')
self.PgnFile1 = pgn.load(self.f1)
self.f2 = protoopen('gamefiles/bilbao.pgn')
self.PgnFile2 = pgn.load(self.f2)
self.f3 = protoopen('gamefiles/material.pgn')
self.PgnFile3 = pgn.load(self.f3)
def get_count(self, filename):
subdir = "puzzles" if self.progress_file.endswith("puzzles.json") else "lessons"
if filename.lower().endswith(".pgn"):
chessfile = PGNFile(protoopen(addDataPrefix("learn/%s/%s" % (subdir, filename))))
chessfile.limit = 1000
chessfile.init_tag_database()
elif filename.lower().endswith(".olv"):
chessfile = OLVFile(protoopen(addDataPrefix("learn/%s/%s" % (subdir, filename)), encoding="utf-8"))
chessfile.close()
count = chessfile.count
return count
def setUp(self):
self.f1 = protoopen('gamefiles/3fold.pgn')
self.PgnFile1 = pgn.load(self.f1)
self.PgnFile1.get_records()
self.f2 = protoopen('gamefiles/bilbao.pgn')
self.PgnFile2 = pgn.load(self.f2)
self.PgnFile2.get_records()
self.f3 = protoopen('gamefiles/material.pgn')
self.PgnFile3 = pgn.load(self.f3)
self.PgnFile3.get_records()
def setUpClass(cls):
cls.f1 = protoopen('gamefiles/3fold.pgn')
cls.PgnFile1 = pgn.load(cls.f1)
cls.PgnFile1.get_records()
cls.f2 = protoopen('gamefiles/bilbao.pgn')
cls.PgnFile2 = pgn.load(cls.f2)
cls.PgnFile2.get_records()
cls.f3 = protoopen('gamefiles/material.pgn')
cls.PgnFile3 = pgn.load(cls.f3)
cls.PgnFile3.get_records()
def start_lesson_from(filename, index=None):
if filename.startswith("lichess_study"):
chessfile = PGNFile(
protoopen(addDataPrefix("learn/lessons/%s" % filename),
encoding="utf-8"))
else:
chessfile = PGNFile(
protoopen(addDataPrefix("learn/lessons/%s" % filename)))
chessfile.limit = 1000
chessfile.init_tag_database()
records, plys = chessfile.get_records()
progress = lessons_solving_progress.get(filename, [0] * chessfile.count)
if index is None:
index = progress.index(0)
rec = records[index]
timemodel = TimeModel(0, 0)
gamemodel = LearnModel(timemodel)
gamemodel.set_learn_data(LESSON, filename, index, len(records))
chessfile.loadToModel(rec, -1, gamemodel)
color = gamemodel.boards[0].color
player_name = conf.get("firstName", _("You"))
w_name = player_name if color == WHITE else "PyChess"
b_name = "PyChess" if color == WHITE else player_name
p0 = (LOCAL, Human, (WHITE, w_name), w_name)
p1 = (LOCAL, Human, (BLACK, b_name), b_name)
def learn_success(gamemodel):
chessfile.loadToModel(rec, -1, gamemodel)
progress = lessons_solving_progress[gamemodel.source]
progress[gamemodel.current_index] = 1
lessons_solving_progress[gamemodel.source] = progress
asyncio. async (gamemodel.restart_analyzer(HINT))
gamemodel.connect("learn_success", learn_success)
def on_game_started(gamemodel):
perspective.activate_panel("annotationPanel")
asyncio. async (gamemodel.start_analyzer(
HINT, force_engine=discoverer.getEngineLearn()))
gamemodel.connect("game_started", on_game_started)
gamemodel.status = WAITING_TO_START
perspective = perspective_manager.get_perspective("games")
asyncio. async (perspective.generalStart(gamemodel, p0, p1))
def opening():
if filename.endswith(".pgn"):
chessfile = pgn.load(protoopen(filename), self.progressbar1)
elif filename.endswith(".epd"):
chessfile = epd.load(protoopen(filename))
elif filename.endswith(".fen"):
chessfile = fen.load(protoopen(filename))
else:
chessfile = None
GLib.idle_add(self.spinner.stop)
GLib.idle_add(self.progress_dialog.hide)
if chessfile is not None:
GLib.idle_add(self.emit, "chessfile_opened", chessfile)
def opening():
if filename.endswith(".pgn"):
chessfile = pgn.load(protoopen(filename), self.progressbar1)
elif filename.endswith(".epd"):
chessfile = epd.load(protoopen(filename))
elif filename.endswith(".fen"):
chessfile = fen.load(protoopen(filename))
else:
chessfile = None
GLib.idle_add(self.spinner.stop)
GLib.idle_add(self.progress_dialog.hide)
if chessfile is not None:
self.chessfile = chessfile
self.chessfiles.append(chessfile)
GLib.idle_add(self.emit, "chessfile_opened0", chessfile)
def pgn_test(self, name):
pgnfile = load(protoopen("gamefiles/%s.pgn" % name))
pgnfile.limit = 1000
pgnfile.init_tag_database()
games, plys = pgnfile.get_records()
for i, game in enumerate(games):
print("%s/%s" % (i + 1, pgnfile.get_count()))
orig_moves_text = normalize(pgnfile.get_movetext(game))
model = pgnfile.loadToModel(game)
print(model.tags["Site"])
new_moves = []
walk(model.boards[0].board, new_moves, model)
new_moves_text = normalize(" ".join(new_moves))
for orig, new in zip(orig_moves_text.split(), new_moves_text.split()):
# Seems most .PGN unnecessary contains unambiguous notation
# when second move candidate is invalid (leaves king in check)
# f.e.: 1.e4 e5 2.d4 Nf6 3.Nc3 Bb4 Nge2
if len(orig) == len(new) + 1 and orig[0] == new[0] and orig[2:] == new[1:]:
continue
elif orig[-1] in "?!" and new[-1] not in "?!":
# pgn export format uses nag
break
elif (orig == "0-0" and new == "O-O") or (orig == "0-0-0" and new == "O-O-O"):
continue
self.assertEqual(orig, new)
pgnfile.close()
def row_activated(self, widget, path, col):
if path is None:
return
filename = addDataPrefix("lectures/%s" % LESSONS[path[0]][0])
chessfile = PGNFile(protoopen(filename))
self.importer = PgnImport(chessfile)
chessfile.init_tag_database(self.importer)
records, plys = chessfile.get_records()
rec = records[random.randrange(0, len(records))]
print(rec)
timemodel = TimeModel(0, 0)
gamemodel = GameModel(timemodel)
gamemodel.set_lesson_game()
chessfile.loadToModel(rec, -1, gamemodel)
name = conf.get("firstName", _("You"))
p0 = (LOCAL, Human, (WHITE, name), name)
name = "pychessbot"
p1 = (LOCAL, Human, (BLACK, name), name)
gamemodel.status = WAITING_TO_START
perspective = perspective_manager.get_perspective("games")
asyncio. async (perspective.generalStart(gamemodel, p0, p1))
def on_file_activated (self, *args):
filename = self._retrieve_filename()
if filename:
if filename == self.get_filename():
return
self.set_filename(filename)
elif self.get_filename():
filename = self.get_filename()
else:
return
if os.path.isdir(filename):
return
ending = filename[filename.rfind(".")+1:]
loader = self.enddir[ending]
self.chessfile = chessfile = loader.load(protoopen(filename))
self.list.get_model().clear()
for gameno in range(len(chessfile)):
names = chessfile.get_player_names (gameno)
names = [ellipsize (name, 9) for name in names]
result = reprResult[chessfile.get_result (gameno)]
result = result.replace("1/2","½")
self.list.get_model().append (names+[result])
self.lastSel = -1 # The row that was last selected
self.list.set_cursor((0,))
def onFileActivated(self, filename):
# filename is None if a non-existent file is passed as command line argument
if filename is None:
return
self.set_filename(filename)
if os.path.isdir(filename):
return
ending = filename[filename.rfind(".") + 1:]
loader = self.enddir[ending]
self.chessfile = chessfile = loader.load(protoopen(filename))
self.list.get_model().clear()
for gameno in range(len(chessfile)):
names = chessfile.get_player_names(gameno)
names = [ellipsize(name, 9) for name in names]
result = reprResult[chessfile.get_result(gameno)]
result = result.replace("1/2", "½")
self.list.get_model().append(["%s." % (gameno + 1)] + names +
[result])
self.last_sel = -1 # The row that was last selected
self.list.set_cursor((0, ))
self.widgets["whitePlayerCombobox"].set_active(0)
self.widgets["blackPlayerCombobox"].set_active(0)
def start_lesson_from(filename, index=None):
chessfile = PGNFile(protoopen(addDataPrefix("learn/lessons/%s" %
filename)))
chessfile.limit = 1000
chessfile.init_tag_database()
records, plys = chessfile.get_records()
progress = lessons_solving_progress.get(filename, [0] * chessfile.count)
if index is None:
index = progress.index(0)
rec = records[index]
timemodel = TimeModel(0, 0)
gamemodel = LearnModel(timemodel)
chessfile.loadToModel(rec, -1, gamemodel)
if len(gamemodel.moves) > 0:
start_lesson_game(gamemodel, filename, chessfile, records, index, rec)
else:
start_puzzle_game(gamemodel,
filename,
records,
index,
rec,
from_lesson=True)
def on_file_activated(self, *args):
filename = self._retrieve_filename()
if filename:
if filename == self.get_filename():
return
self.set_filename(filename)
elif self.get_filename():
filename = self.get_filename()
else:
return
if os.path.isdir(filename):
return
ending = filename[filename.rfind(".") + 1:]
loader = self.enddir[ending]
self.chessfile = chessfile = loader.load(protoopen(filename))
self.list.get_model().clear()
for gameno in range(len(chessfile)):
names = chessfile.get_player_names(gameno)
names = [ellipsize(name, 9) for name in names]
result = reprResult[chessfile.get_result(gameno)]
result = result.replace("1/2", "½")
self.list.get_model().append(names + [result])
self.lastSel = -1 # The row that was last selected
self.list.set_cursor((0, ))
def onFileActivated(self, filename):
# filename is None if a non-existent file is passed as command line argument
if filename is None:
return
self.set_filename(filename)
if os.path.isdir(filename):
return
ending = filename[filename.rfind(".") + 1:]
loader = self.enddir[ending]
self.chessfile = chessfile = loader.load(protoopen(filename))
self.list.get_model().clear()
for gameno in range(len(chessfile)):
names = chessfile.get_player_names(gameno)
names = [ellipsize(name, 9) for name in names]
result = reprResult[chessfile.get_result(gameno)]
result = result.replace("1/2", "½")
self.list.get_model().append(["%s." % (gameno + 1)] + names +
[result])
self.last_sel = -1 # The row that was last selected
self.list.set_cursor((0, ))
self.widgets["whitePlayerCombobox"].set_active(0)
self.widgets["blackPlayerCombobox"].set_active(0)
def opening():
# Redirection of the PGN file
nonlocal filename
for ext in [".sqlite", ".bin", ".scout"]:
if filename.endswith(ext):
filename = filename[:len(filename) - len(ext)] + ".pgn"
# Processing by file extension
if filename.endswith(".pgn"):
GLib.idle_add(self.progressbar.show)
GLib.idle_add(self.progressbar.set_text,
_("Opening chessfile..."))
chessfile = PGNFile(protoopen(filename), self.progressbar)
self.importer = chessfile.init_tag_database()
if self.importer is not None and self.importer.cancel:
chessfile.tag_database.close()
if os.path.isfile(chessfile.sqlite_path):
os.remove(chessfile.sqlite_path)
chessfile = None
else:
chessfile.init_scoutfish()
chessfile.init_chess_db()
elif filename.endswith(".epd"):
self.importer = None
chessfile = epd.load(protoopen(filename))
elif filename.endswith(".olv"):
self.importer = None
chessfile = olv.load(protoopen(filename, encoding="utf-8"))
elif filename.endswith(".fen"):
self.importer = None
chessfile = fen.load(protoopen(filename))
else:
self.importer = None
chessfile = None
GLib.idle_add(self.spinner.stop)
GLib.idle_add(self.spinner.hide)
GLib.idle_add(self.progress_dialog.hide)
if chessfile is not None:
self.chessfile = chessfile
self.chessfiles.append(chessfile)
GLib.idle_add(self.emit, "chessfile_opened0", chessfile)
else:
if self.chessfile is None:
self.close(None)
def open_chessfile(self, filename):
if filename.endswith(".pdb"):
chessfile = database.load(filename)
elif filename.endswith(".pgn"):
chessfile = pgn.load(protoopen(filename))
elif filename.endswith(".epd"):
chessfile = epd.load(protoopen(filename))
elif filename.endswith(".fen"):
chessfile = fen.load(protoopen(filename))
else:
return
if self.gamelist is None:
self.init_layout()
perspective_manager.activate_perspective("database")
self.emit("chessfile_opened", chessfile)
def loadAndStart(self, uri, loader, gameno, position, first_time=True):
if first_time:
assert self.status == WAITING_TO_START
uriIsFile = not isinstance(uri, str)
if not uriIsFile:
chessfile = loader.load(protoopen(uri))
else:
chessfile = loader.load(uri)
self.gameno = gameno
self.emit("game_loading", uri)
try:
chessfile.loadToModel(gameno, -1, self)
# Postpone error raising to make games loadable to the point of the
# error
except LoadingError as e:
error = e
else:
error = None
if self.players:
self.players[WHITE].setName(self.tags["White"])
self.players[BLACK].setName(self.tags["Black"])
self.emit("game_loaded", uri)
self.needsSave = False
if not uriIsFile:
self.uri = uri
else:
self.uri = None
# Even if the game "starts ended", the players should still be moved
# to the last position, so analysis is correct, and a possible "undo"
# will work as expected.
for spectator in self.spectators.values():
spectator.setOptionInitialBoard(self)
for player in self.players:
player.setOptionInitialBoard(self)
if self.timed:
self.timemodel.setMovingColor(self.boards[-1].color)
if first_time:
if self.status == RUNNING:
if self.timed:
self.timemodel.start()
# Store end status from Result tag
if self.status in (DRAW, WHITEWON, BLACKWON):
self.endstatus = self.status
self.status = WAITING_TO_START
self.start()
if error:
raise error
def loadAndStart(self, uri, loader, gameno, position, first_time=True):
if first_time:
assert self.status == WAITING_TO_START
uriIsFile = not isinstance(uri, str)
if not uriIsFile:
chessfile = loader.load(protoopen(uri))
else:
chessfile = loader.load(uri)
self.gameno = gameno
self.emit("game_loading", uri)
try:
chessfile.loadToModel(gameno, -1, self)
# Postpone error raising to make games loadable to the point of the
# error
except LoadingError as e:
error = e
else:
error = None
if self.players:
self.players[WHITE].setName(self.tags["White"])
self.players[BLACK].setName(self.tags["Black"])
self.emit("game_loaded", uri)
self.needsSave = False
if not uriIsFile:
self.uri = uri
else:
self.uri = None
# Even if the game "starts ended", the players should still be moved
# to the last position, so analysis is correct, and a possible "undo"
# will work as expected.
for spectator in self.spectators.values():
spectator.setOptionInitialBoard(self)
for player in self.players:
player.setOptionInitialBoard(self)
if self.timed:
self.timemodel.setMovingColor(self.boards[-1].color)
if first_time:
if self.status == RUNNING:
if self.timed:
self.timemodel.start()
# Store end status from Result tag
if self.status in (DRAW, WHITEWON, BLACKWON):
self.endstatus = self.status
self.status = WAITING_TO_START
self.start()
if error:
raise error
def opening():
if filename.endswith(".pgn"):
GLib.idle_add(self.progressbar.show)
GLib.idle_add(self.progressbar.set_text,
_("Opening chessfile..."))
chessfile = PGNFile(protoopen(filename), self.progressbar)
self.importer = chessfile.init_tag_database()
if self.importer is not None and self.importer.cancel:
chessfile.tag_database.close()
if os.path.isfile(chessfile.sqlite_path):
os.remove(chessfile.sqlite_path)
chessfile = None
else:
chessfile.init_scoutfish()
chessfile.init_chess_db()
elif filename.endswith(".epd"):
self.importer = None
chessfile = epd.load(protoopen(filename))
elif filename.endswith(".olv"):
self.importer = None
chessfile = olv.load(protoopen(filename, encoding="utf-8"))
elif filename.endswith(".fen"):
self.importer = None
chessfile = fen.load(protoopen(filename))
else:
self.importer = None
chessfile = None
GLib.idle_add(self.spinner.stop)
GLib.idle_add(self.spinner.hide)
GLib.idle_add(self.progress_dialog.hide)
if chessfile is not None:
self.chessfile = chessfile
self.chessfiles.append(chessfile)
GLib.idle_add(self.emit, "chessfile_opened0", chessfile)
else:
if self.chessfile is None:
self.close(None)
def opening():
if filename.endswith(".pgn"):
GLib.idle_add(self.progressbar.show)
GLib.idle_add(self.progressbar.set_text, _("Opening chessfile..."))
chessfile = PGNFile(protoopen(filename), self.progressbar)
self.importer = chessfile.init_tag_database()
if self.importer is not None and self.importer.cancel:
chessfile.tag_database.close()
if os.path.isfile(chessfile.sqlite_path):
os.remove(chessfile.sqlite_path)
chessfile = None
else:
chessfile.init_scoutfish()
chessfile.init_chess_db()
elif filename.endswith(".epd"):
self.importer = None
chessfile = epd.load(protoopen(filename))
elif filename.endswith(".olv"):
self.importer = None
chessfile = olv.load(protoopen(filename, encoding="utf-8"))
elif filename.endswith(".fen"):
self.importer = None
chessfile = fen.load(protoopen(filename))
else:
self.importer = None
chessfile = None
GLib.idle_add(self.spinner.stop)
GLib.idle_add(self.spinner.hide)
GLib.idle_add(self.progress_dialog.hide)
if chessfile is not None:
self.chessfile = chessfile
self.chessfiles.append(chessfile)
GLib.idle_add(self.emit, "chessfile_opened0", chessfile)
else:
if self.chessfile is None:
self.close(None)
def start_puzzle_from(filename, index=None):
if filename.lower().endswith(".pgn"):
chessfile = PGNFile(protoopen(addDataPrefix("learn/puzzles/%s" % filename)))
chessfile.limit = 1000
chessfile.init_tag_database()
elif filename.lower().endswith(".olv"):
chessfile = OLVFile(protoopen(addDataPrefix("learn/puzzles/%s" % filename), encoding="utf-8"))
records, plys = chessfile.get_records()
progress = puzzles_solving_progress.get(filename, [0] * chessfile.count)
if index is None:
index = progress.index(0)
rec = records[index]
timemodel = TimeModel(0, 0)
gamemodel = LearnModel(timemodel)
chessfile.loadToModel(rec, 0, gamemodel)
start_puzzle_game(gamemodel, filename, records, index, rec)
def loadAndStart(self, uri, loader, gameno, position):
assert self.status == WAITING_TO_START
uriIsFile = type(uri) != str
if not uriIsFile:
chessfile = loader.load(protoopen(uri))
else:
chessfile = loader.load(uri)
self.emit("game_loading", uri)
try:
chessfile.loadToModel(gameno, position, self)
#Postpone error raising to make games loadable to the point of the error
except LoadingError, e:
error = e
def loadAndStart (self, uri, loader, gameno, position):
assert self.status == WAITING_TO_START
uriIsFile = type(uri) != str
if not uriIsFile:
chessfile = loader.load(protoopen(uri))
else:
chessfile = loader.load(uri)
self.emit("game_loading", uri)
try:
chessfile.loadToModel(gameno, position, self)
#Postpone error raising to make games loadable to the point of the error
except LoadingError, e:
error = e
def feed(pgnfile, lang):
cf = load(protoopen(pgnfile))
rows = []
old_eco = ""
ply_max = 0
for i, game in enumerate(cf.games):
model = cf.loadToModel(i)
eco = cf._getTag(i, "ECO")[:3]
opening = cf._getTag(i, "Opening")
if opening is None:
opening = ""
variation = cf._getTag(i, "Variation")
if variation is None:
variation = ""
base = int(old_eco != eco)
ply = len(model.moves)
ply_max = max(ply_max, ply)
if ply == 0:
cu = conn.cursor()
cu.execute(
"select * from openings where eco=? and lang='en' and base=1",
(eco, ))
res = cu.fetchone()
if res is not None:
hash = res[0]
else:
hash = memoryview(hash_struct.pack(
model.boards[-1].board.hash))
if opening:
rows.append((hash, base, unicode(eco), unicode(lang),
unicode(opening), unicode(variation)))
old_eco = eco
c.executemany(
"insert into openings(hash, base, eco, lang, opening, variation) values (?, ?, ?, ?, ?, ?)",
rows)
conn.commit()
print("Max ply was %s" % ply_max)
def feed(pgnfile, lang):
cf = load(protoopen(pgnfile))
cf.limit = 5000
importer = PgnImport(cf)
cf.init_tag_database(importer)
records, plys = cf.get_records()
rows = []
old_eco = ""
for rec in records:
model = cf.loadToModel(rec)
eco = rec["ECO"]
opening = rec["White"]
if opening is None:
opening = ""
variation = rec["Black"]
if variation is None:
variation = ""
base = int(old_eco != eco)
ply = len(model.moves)
if ply == 0:
cu = conn.cursor()
cu.execute(
"select * from openings where eco=? and lang='en' and base=1",
(eco, ))
res = cu.fetchone()
if res is not None:
hash = res[0]
else:
hash = memoryview(hash_struct.pack(
model.boards[-1].board.hash))
if opening:
rows.append((hash, base, eco, lang, opening, variation))
old_eco = eco
c.executemany(
"insert into openings(hash, base, eco, lang, opening, variation) values (?, ?, ?, ?, ?, ?)",
rows)
conn.commit()
def start_lesson_from(filename, index=None):
chessfile = PGNFile(protoopen(addDataPrefix("learn/lessons/%s" % filename)))
chessfile.limit = 1000
chessfile.init_tag_database()
records, plys = chessfile.get_records()
progress = lessons_solving_progress.get(filename, [0] * chessfile.count)
if index is None:
index = progress.index(0)
rec = records[index]
timemodel = TimeModel(0, 0)
gamemodel = LearnModel(timemodel)
gamemodel.set_learn_data(LESSON, filename, index, len(records))
chessfile.loadToModel(rec, -1, gamemodel)
color = gamemodel.boards[0].color
player_name = conf.get("firstName", _("You"))
w_name = player_name if color == WHITE else "PyChess"
b_name = "PyChess" if color == WHITE else player_name
p0 = (LOCAL, Human, (WHITE, w_name), w_name)
p1 = (LOCAL, Human, (BLACK, b_name), b_name)
def learn_success(gamemodel):
progress = lessons_solving_progress[gamemodel.source]
progress[gamemodel.current_index] = 1
lessons_solving_progress[gamemodel.source] = progress
asyncio.async(gamemodel.restart_analyzer(HINT))
gamemodel.connect("learn_success", learn_success)
def start_analyzer(gamemodel):
asyncio.async(gamemodel.start_analyzer(HINT, force_engine=discoverer.getEngineLearn()))
gamemodel.connect("game_started", start_analyzer)
gamemodel.status = WAITING_TO_START
perspective = perspective_manager.get_perspective("games")
asyncio.async(perspective.generalStart(gamemodel, p0, p1))
def queryGameno(path):
pgnfile = pgn.load(protoopen(path))
print "Selected file %s" % path
if len(pgnfile) == 0:
print "The file is empty."
sys.exit()
print
print "The file contains the following games:"
for i in xrange(len(pgnfile)):
name1, name2 = pgnfile.get_player_names(i)
print "[%d] %s vs. %s" % (i, name1, name2)
print
if len(pgnfile) == 1:
print "Autoselecting game 0."
gameno = 0
else:
gameno = int(raw_input("What engine should be your analyzer? [n] "))
print
return pgnfile, gameno
def queryGameno(path):
pgnfile = pgn.load(protoopen(path))
print "Selected file %s" % path
if len(pgnfile) == 0:
print "The file is empty."
sys.exit()
print
print "The file contains the following games:"
for i in xrange(len(pgnfile)):
name1, name2 = pgnfile.get_player_names(i)
print "[%d] %s vs. %s" % (i, name1, name2)
print
if len(pgnfile) == 1:
print "Autoselecting game 0."
gameno = 0
else:
gameno = int(raw_input("What engine should be your analyzer? [n] "))
print
return pgnfile, gameno
def queryGameno(path):
pgnfile = pgn.load(protoopen(path))
print("Selected file %s" % path)
if len(pgnfile) == 0:
print("The file is empty.")
sys.exit()
print()
print("The file contains the following games:")
for i in range(len(pgnfile)):
name1, name2 = pgnfile.get_player_names(i)
print("[%d] %s vs. %s" % (i, name1, name2))
print()
if len(pgnfile) == 1:
print("Autoselecting game 0.")
gameno = 0
else:
gameno = int(raw_input("Select game number to be analyzed. [n]: "))
print()
return pgnfile, gameno
def queryGameno(path):
pgnfile = pgn.load(protoopen(path))
print("Selected file %s" % path)
if len(pgnfile) == 0:
print("The file is empty.")
sys.exit()
print()
print("The file contains the following games:")
for i in range(len(pgnfile)):
name1, name2 = pgnfile.get_player_names(i)
print("[%d] %s vs. %s" % (i, name1, name2))
print()
if len(pgnfile) == 1:
print("Autoselecting game 0.")
gameno = 0
else:
gameno = int(raw_input("Select game number to be analyzed. [n]: "))
print()
return pgnfile, gameno
def start_puzzle_from(filename):
chessfile = PGNFile(protoopen(addDataPrefix("lectures/%s" % filename)))
chessfile.limit = 1000
importer = PgnImport(chessfile)
chessfile.init_tag_database(importer)
records, plys = chessfile.get_records()
rec = records[random.randrange(0, len(records))]
timemodel = TimeModel(0, 0)
gamemodel = GameModel(timemodel)
gamemodel.set_practice_game()
gamemodel.practice = ("puzzle", filename)
chessfile.loadToModel(rec, 0, gamemodel)
# TODO: change colors according to FEN!
name = rec["White"]
p0 = (LOCAL, Human, (WHITE, name), name)
engine = discoverer.getEngineByName(stockfish_name)
name = rec["Black"]
ponder_off = True
p1 = (ARTIFICIAL, discoverer.initPlayerEngine,
(engine, BLACK, 20, variants[NORMALCHESS], 60, 0, 0,
ponder_off), name)
def fix_name(gamemodel, name):
asyncio. async (gamemodel.start_analyzer(HINT,
force_engine=stockfish_name))
gamemodel.players[1].name = name
gamemodel.emit("players_changed")
gamemodel.connect("game_started", fix_name, name)
gamemodel.variant.need_initial_board = True
gamemodel.status = WAITING_TO_START
perspective = perspective_manager.get_perspective("games")
asyncio. async (perspective.generalStart(gamemodel, p0, p1))
def feed(pgnfile, lang):
cf = load(protoopen(pgnfile))
cf.limit = 5000
cf.init_tag_database()
records, plys = cf.get_records()
rows = []
old_eco = ""
for rec in records:
model = cf.loadToModel(rec)
eco = rec["ECO"]
opening = rec["White"]
if opening is None:
opening = ""
variation = rec["Black"]
if variation is None:
variation = ""
base = int(old_eco != eco)
ply = len(model.moves)
if ply == 0:
cu = conn.cursor()
cu.execute("select * from openings where eco=? and lang='en' and base=1", (eco,))
res = cu.fetchone()
if res is not None:
hash = res[0]
else:
hash = memoryview(hash_struct.pack(model.boards[-1].board.hash))
if opening:
rows.append((hash, base, eco, lang, opening, variation))
old_eco = eco
c.executemany("insert into openings(hash, base, eco, lang, opening, variation) values (?, ?, ?, ?, ?, ?)", rows)
conn.commit()
def feed(pgnfile, lang):
cf = load(protoopen(pgnfile))
rows = []
old_eco = ""
ply_max = 0
for i, game in enumerate(cf.games):
model = cf.loadToModel(i)
eco = cf._getTag(i, "ECO")[:3]
opening = cf._getTag(i, "Opening")
if opening is None:
opening = ""
variation = cf._getTag(i, "Variation")
if variation is None:
variation = ""
base = int(old_eco != eco)
ply = len(model.moves)
ply_max = max(ply_max, ply)
if ply == 0:
cu = conn.cursor()
cu.execute("select * from openings where eco=? and lang='en' and base=1", (eco,))
res = cu.fetchone()
if res is not None:
hash = res[0]
else:
hash = memoryview(hash_struct.pack(model.boards[-1].board.hash))
if opening:
rows.append((hash, base, unicode(eco), unicode(lang), unicode(opening), unicode(variation)))
old_eco = eco
c.executemany("insert into openings(hash, base, eco, lang, opening, variation) values (?, ?, ?, ?, ?, ?)", rows)
conn.commit()
print("Max ply was %s" % ply_max)
def start_lesson_from(filename, index=None):
chessfile = PGNFile(protoopen(addDataPrefix("learn/lessons/%s" % filename)))
chessfile.limit = 1000
chessfile.init_tag_database()
records, plys = chessfile.get_records()
progress = lessons_solving_progress.get(filename, [0] * chessfile.count)
if index is None:
index = progress.index(0)
rec = records[index]
timemodel = TimeModel(0, 0)
gamemodel = LearnModel(timemodel)
chessfile.loadToModel(rec, -1, gamemodel)
if len(gamemodel.moves) > 0:
start_lesson_game(gamemodel, filename, chessfile, records, index, rec)
else:
start_puzzle_game(gamemodel, filename, records, index, rec, from_lesson=True)
def feed(pgnfile, lang):
# Check the existence of the file
if not os.path.isfile(pgnfile):
return
# Load the ECO file first
print(' - Parsing')
cf = load(protoopen(pgnfile))
cf.limit = 5000
cf.init_tag_database()
records, plys = cf.get_records()
# Cache the content
entries = []
plyMax = 0
old_eco = ""
for rec in records:
model = cf.loadToModel(rec)
eco = '' if rec['ECO'] is None else rec['ECO']
entry = {
'h': [], # Hashes
'f': '', # Final hash of the line
'n': [], # FENs
'm': old_eco !=
eco, # Main line = shortest sequence of moves for the ECO code. The 'EN' ECO file is specially crafted
'e': eco, # ECO
'o': '' if rec['White'] is None else rec['White'], # Opening
'v': '' if rec['Black'] is None else rec['Black'], # Variation
'p': len(model.moves)
} # Number of plies
plyMax = max(plyMax, entry['p'])
# No move means that we are translating the name of the ECO code, so we need to find all the related positions from another language
if entry['p'] == 0:
if lang == ECO_MAIN_LANG:
continue
c.execute(
"select hash, endline, fen from openings where eco=? and lang=? and mainline=1",
(eco, ECO_MAIN_LANG))
rows = c.fetchall()
for row in rows:
entry['h'].append(row[0])
if (row[1] == int(True)):
entry['f'] = row[0]
entry['n'].append(row[2])
else:
# Find the Polyglot hash for each position of the opening
for i in range(entry['p']):
nextboard = model.getBoardAtPly(i, 0).board.next
h = hex(nextboard.hash)[2:]
entry['h'].append(h)
entry['f'] = h
entry['n'].append(nextboard.asFen())
entries.append(entry)
old_eco = entry['e']
print(' - Max ply : %d' % plyMax)
# Process all the data in reverse order
for depth in reversed(range(plyMax + 1)):
sys.stdout.write(
"\r - Loading into the database (%d remaining) " % depth)
sys.stdout.flush()
for i in reversed(range(len(
entries))): # Long lines are overwritten by short lines
entry = entries[i]
if entry['p'] != depth:
continue
for i in range(len(entry['h'])):
h = entry['h'][i]
hkey = int(h[-2:], 16)
c.execute(
"select endline from openings where hash=? and hkey=? and lang=?",
(h, hkey, lang))
r = c.fetchone()
if r is not None and r[0] == int(True):
continue
c.execute(
"delete from openings where hash=? and hkey=? and lang=?",
(h, hkey, lang))
c.execute(
"insert into openings (hash, hkey, mainline, endline, eco, lang, opening, variation, fen) values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
(h, hkey, int(
entry['m']), int(h == entry['f']), entry['e'],
lang, entry['o'], entry['v'], entry['n'][i]))
conn.commit()
print('\n - Processed %d openings' % len(entries))
def do_import(self, filename, info=None, progressbar=None):
DB_MAXINT_SHIFT = get_maxint_shift(self.engine)
self.progressbar = progressbar
orig_filename = filename
count_source = self.conn.execute(
self.count_source.where(source.c.name == orig_filename)).scalar()
if count_source > 0:
print("%s is already imported" % filename)
return
# collect new names not in they dict yet
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
self.source_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
self.bitboard_data = []
self.stat_ins_data = []
self.stat_upd_data = []
self.tag_game_data = []
if filename.startswith("http"):
filename = download_file(filename, progressbar=progressbar)
if filename is None:
return
else:
if not os.path.isfile(filename):
print("Can't open %s" % filename)
return
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
zf = zipfile.ZipFile(filename, "r")
files = [f for f in zf.namelist() if f.lower().endswith(".pgn")]
else:
zf = None
files = [filename]
for pgnfile in files:
basename = os.path.basename(pgnfile)
if progressbar is not None:
GLib.idle_add(progressbar.set_text,
"Reading %s ..." % basename)
else:
print("Reading %s ..." % pgnfile)
if zf is None:
size = os.path.getsize(pgnfile)
handle = protoopen(pgnfile)
else:
size = zf.getinfo(pgnfile).file_size
handle = io.TextIOWrapper(zf.open(pgnfile),
encoding=PGN_ENCODING,
newline='')
cf = PgnBase(handle, [])
# estimated game count
all_games = max(size / 840, 1)
self.CHUNK = 1000 if all_games > 5000 else 100
get_id = self.get_id
# use transaction to avoid autocommit slowness
trans = self.conn.begin()
try:
i = 0
for tagtext, movetext in read_games(handle):
tags = defaultdict(str, tagre.findall(tagtext))
if not tags:
print("Empty game #%s" % (i + 1))
continue
if self.cancel:
trans.rollback()
return
fenstr = tags.get("FEN")
variant = tags.get("Variant")
if variant:
if "fischer" in variant.lower() or "960" in variant:
variant = "Fischerandom"
else:
variant = variant.lower().capitalize()
# Fixes for some non statndard Chess960 .pgn
if fenstr and variant == "Fischerandom":
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
if variant not in name2variant:
print("Unknown variant: %s" % variant)
continue
variant = name2variant[variant].variant
if variant == NORMALCHESS:
# lichess uses tag [Variant "Standard"]
variant = 0
board = START_BOARD.clone()
else:
board = LBoard(variant)
elif fenstr:
variant = 0
board = LBoard()
else:
variant = 0
board = START_BOARD.clone()
if fenstr:
try:
board.applyFen(fenstr)
except SyntaxError as e:
print(
_("The game #%s can't be loaded, because of an error parsing FEN"
) % (i + 1), e.args[0])
continue
elif variant:
board.applyFen(FEN_START)
movelist = array("H")
comments = []
cf.error = None
# First we try to use simple_parse_movetext()
# assuming most games in .pgn contains only moves
# without any comments/variations
simple = False
if not fenstr and not variant:
bitboards = []
simple = cf.simple_parse_movetext(
movetext, board, movelist, bitboards)
if cf.error is not None:
print("ERROR in %s game #%s" % (pgnfile, i + 1),
cf.error.args[0])
continue
# If simple_parse_movetext() find any comments/variations
# we restart parsing with full featured parse_movetext()
if not simple:
movelist = array("H")
bitboards = None
# in case simple_parse_movetext failed we have to reset our lboard
if not fenstr and not variant:
board = START_BOARD.clone()
# parse movetext to create boards tree structure
boards = [board]
boards = cf.parse_movetext(movetext,
boards[0],
-1,
pgn_import=True)
if cf.error is not None:
print("ERROR in %s game #%s" % (pgnfile, i + 1),
cf.error.args[0])
continue
# create movelist and comments from boards tree
walk(boards[0], movelist, comments)
white = tags.get('White')
black = tags.get('Black')
if not movelist:
if (not comments) and (not white) and (not black):
print("Empty game #%s" % (i + 1))
continue
event_id = get_id(tags.get('Event'), event, EVENT)
site_id = get_id(tags.get('Site'), site, SITE)
game_date = tags.get('Date').strip()
try:
if game_date and '?' not in game_date:
ymd = game_date.split('.')
if len(ymd) == 3:
game_year, game_month, game_day = map(int, ymd)
else:
game_year, game_month, game_day = int(
game_date[:4]), None, None
elif game_date and '?' not in game_date[:4]:
game_year, game_month, game_day = int(
game_date[:4]), None, None
else:
game_year, game_month, game_day = None, None, None
except:
game_year, game_month, game_day = None, None, None
game_round = tags.get('Round')
white_fide_id = tags.get('WhiteFideId')
black_fide_id = tags.get('BlackFideId')
white_id = get_id(unicode(white),
player,
PLAYER,
fide_id=white_fide_id)
black_id = get_id(unicode(black),
player,
PLAYER,
fide_id=black_fide_id)
result = tags.get("Result")
if result in pgn2Const:
result = pgn2Const[result]
else:
print("Invalid Result tag in game #%s: %s" %
(i + 1, result))
continue
white_elo = tags.get('WhiteElo')
white_elo = int(
white_elo
) if white_elo and white_elo.isdigit() else None
black_elo = tags.get('BlackElo')
black_elo = int(
black_elo
) if black_elo and black_elo.isdigit() else None
time_control = tags.get("TimeControl")
eco = tags.get("ECO")
eco = eco[:3] if eco else None
fen = tags.get("FEN")
board_tag = tags.get("Board")
annotator_id = get_id(tags.get("Annotator"), annotator,
ANNOTATOR)
source_id = get_id(unicode(orig_filename),
source,
SOURCE,
info=info)
game_id = self.next_id[GAME]
self.next_id[GAME] += 1
# annotated game
if bitboards is None:
for ply, board in enumerate(boards):
if ply == 0:
continue
bb = board.friends[0] | board.friends[1]
# Avoid to include mate in x .pgn collections and similar in opening tree
if fen and "/pppppppp/8/8/8/8/PPPPPPPP/" not in fen:
ply = -1
self.bitboard_data.append({
'game_id':
game_id,
'ply':
ply,
'bitboard':
bb - DB_MAXINT_SHIFT,
})
if ply <= STAT_PLY_MAX:
self.stat_ins_data.append({
'ply': ply,
'bitboard': bb - DB_MAXINT_SHIFT,
'count': 0,
'whitewon': 0,
'blackwon': 0,
'draw': 0,
'white_elo_count': 0,
'black_elo_count': 0,
'white_elo': 0,
'black_elo': 0,
})
self.stat_upd_data.append({
'_ply':
ply,
'_bitboard':
bb - DB_MAXINT_SHIFT,
'_count':
1,
'_whitewon':
1 if result == WHITEWON else 0,
'_blackwon':
1 if result == BLACKWON else 0,
'_draw':
1 if result == DRAW else 0,
'_white_elo_count':
1 if white_elo is not None else 0,
'_black_elo_count':
1 if black_elo is not None else 0,
'_white_elo':
white_elo if white_elo is not None else 0,
'_black_elo':
black_elo if black_elo is not None else 0,
})
# simple game
else:
for ply, bb in enumerate(bitboards):
if ply == 0:
continue
self.bitboard_data.append({
'game_id':
game_id,
'ply':
ply,
'bitboard':
bb - DB_MAXINT_SHIFT,
})
if ply <= STAT_PLY_MAX:
self.stat_ins_data.append({
'ply': ply,
'bitboard': bb - DB_MAXINT_SHIFT,
'count': 0,
'whitewon': 0,
'blackwon': 0,
'draw': 0,
'white_elo_count': 0,
'black_elo_count': 0,
'white_elo': 0,
'black_elo': 0,
})
self.stat_upd_data.append({
'_ply':
ply,
'_bitboard':
bb - DB_MAXINT_SHIFT,
'_count':
1,
'_whitewon':
1 if result == WHITEWON else 0,
'_blackwon':
1 if result == BLACKWON else 0,
'_draw':
1 if result == DRAW else 0,
'_white_elo_count':
1 if white_elo is not None else 0,
'_black_elo_count':
1 if black_elo is not None else 0,
'_white_elo':
white_elo if white_elo is not None else 0,
'_black_elo':
black_elo if black_elo is not None else 0,
})
ply_count = tags.get("PlyCount")
if not ply_count and not fen:
ply_count = len(
bitboards) if bitboards is not None else len(
boards)
self.game_data.append({
'event_id':
event_id,
'site_id':
site_id,
'date_year':
game_year,
'date_month':
game_month,
'date_day':
game_day,
'round':
game_round,
'white_id':
white_id,
'black_id':
black_id,
'result':
result,
'white_elo':
white_elo,
'black_elo':
black_elo,
'ply_count':
ply_count,
'eco':
eco,
'fen':
fen,
'variant':
variant,
'board':
board_tag,
'time_control':
time_control,
'annotator_id':
annotator_id,
'source_id':
source_id,
'movelist':
movelist.tostring(),
'comments':
unicode("|".join(comments)),
})
i += 1
if len(self.game_data) >= self.CHUNK:
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player,
self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator,
self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source,
self.source_data)
self.source_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if self.bitboard_data:
self.conn.execute(self.ins_bitboard,
self.bitboard_data)
self.bitboard_data = []
self.conn.execute(self.ins_stat,
self.stat_ins_data)
self.conn.execute(self.upd_stat,
self.stat_upd_data)
self.stat_ins_data = []
self.stat_upd_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
i / float(all_games))
GLib.idle_add(
progressbar.set_text,
"%s games from %s imported" % (i, basename))
else:
print(pgnfile, i)
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source, self.source_data)
self.source_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if self.bitboard_data:
self.conn.execute(self.ins_bitboard, self.bitboard_data)
self.bitboard_data = []
self.conn.execute(self.ins_stat, self.stat_ins_data)
self.conn.execute(self.upd_stat, self.stat_upd_data)
self.stat_ins_data = []
self.stat_upd_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
i / float(all_games))
GLib.idle_add(progressbar.set_text,
"%s games from %s imported" % (i, basename))
else:
print(pgnfile, i)
trans.commit()
except SQLAlchemyError as e:
trans.rollback()
print("Importing %s failed! \n%s" % (pgnfile, e))
def start_puzzle_from(filename, index=None):
if filename.lower().endswith(".pgn"):
chessfile = PGNFile(protoopen(addDataPrefix("learn/puzzles/%s" % filename)))
chessfile.limit = 1000
chessfile.init_tag_database()
elif filename.lower().endswith(".olv"):
chessfile = OLVFile(protoopen(addDataPrefix("learn/puzzles/%s" % filename), encoding="utf-8"))
records, plys = chessfile.get_records()
progress = puzzles_solving_progress.get(filename, [0] * chessfile.count)
if index is None:
index = progress.index(0)
rec = records[index]
timemodel = TimeModel(0, 0)
gamemodel = LearnModel(timemodel)
chessfile.loadToModel(rec, 0, gamemodel)
gamemodel.set_learn_data(PUZZLE, filename, index, len(records))
engine = discoverer.getEngineByName(discoverer.getEngineLearn())
ponder_off = True
color = gamemodel.boards[0].color
w_name = "" if rec["White"] is None else rec["White"]
b_name = "" if rec["Black"] is None else rec["Black"]
player_name = conf.get("firstName", _("You"))
engine_name = discoverer.getName(engine)
if rec["Event"].startswith("Lichess Practice"):
w_name = player_name if color == WHITE else engine_name
b_name = engine_name if color == WHITE else player_name
opp_name = engine_name if rec["Event"].startswith("Lichess Practice") else b_name
if color == WHITE:
p0 = (LOCAL, Human, (WHITE, w_name), w_name)
p1 = (ARTIFICIAL, discoverer.initPlayerEngine,
(engine, BLACK, 20, variants[NORMALCHESS], 20, 0, 0, ponder_off), b_name)
else:
p0 = (ARTIFICIAL, discoverer.initPlayerEngine,
(engine, WHITE, 20, variants[NORMALCHESS], 20, 0, 0, ponder_off), w_name)
p1 = (LOCAL, Human, (BLACK, b_name), b_name)
def start_analyzer(gamemodel, name, color):
asyncio.async(gamemodel.start_analyzer(HINT, force_engine=discoverer.getEngineLearn()))
gamemodel.players[1 - color].name = name
gamemodel.emit("players_changed")
gamemodel.connect("game_started", start_analyzer, opp_name, color)
def goal_checked(gamemodle):
if gamemodel.reason == PRACTICE_GOAL_REACHED:
progress = puzzles_solving_progress[gamemodel.source]
progress[gamemodel.current_index] = 1
puzzles_solving_progress[gamemodel.source] = progress
gamemodel.connect("goal_checked", goal_checked)
gamemodel.variant.need_initial_board = True
gamemodel.status = WAITING_TO_START
perspective = perspective_manager.get_perspective("games")
asyncio.async(perspective.generalStart(gamemodel, p0, p1))
def do_import(self, filename, info=None, progressbar=None):
self.progressbar = progressbar
orig_filename = filename
count_source = self.conn.execute(
self.count_source.where(source.c.name == orig_filename)).scalar()
if count_source > 0:
log.info("%s is already imported" % filename)
return
# collect new names not in they dict yet
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
self.source_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
self.tag_game_data = []
if filename.startswith("http"):
filename = download_file(filename, progressbar=progressbar)
if filename is None:
return
else:
if not os.path.isfile(filename):
log.info("Can't open %s" % filename)
return
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
with zipfile.ZipFile(filename, "r") as zf:
path = os.path.dirname(filename)
files = [
os.path.join(path, f) for f in zf.namelist()
if f.lower().endswith(".pgn")
]
zf.extractall(path)
else:
files = [filename]
for pgnfile in files:
base_offset = self.chessfile.size if self.append_pgn else 0
basename = os.path.basename(pgnfile)
if progressbar is not None:
GLib.idle_add(progressbar.set_text,
_("Reading %s ..." % basename))
else:
log.info("Reading %s ..." % pgnfile)
size = os.path.getsize(pgnfile)
handle = protoopen(pgnfile)
# estimated game count
all_games = max(size / 840, 1)
get_id = self.get_id
# use transaction to avoid autocommit slowness
# and to let undo importing (rollback) if self.cancel was set
trans = self.conn.begin()
try:
i = 0
for tags in read_games(handle):
if not tags:
log.info("Empty game #%s" % (i + 1))
continue
if self.cancel:
trans.rollback()
return
fenstr = tags["FEN"]
variant = tags["Variant"]
if variant:
if "fischer" in variant.lower() or "960" in variant:
variant = "Fischerandom"
else:
variant = variant.lower().capitalize()
# Fixes for some non statndard Chess960 .pgn
if fenstr and variant == "Fischerandom":
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
if variant not in name2variant:
log.info("Unknown variant: %s" % variant)
continue
variant = name2variant[variant].variant
if variant == NORMALCHESS:
# lichess uses tag [Variant "Standard"]
variant = 0
else:
variant = 0
if basename == "eco.pgn":
white = tags["Opening"]
black = tags["Variation"]
else:
white = tags["White"]
black = tags["Black"]
event_id = get_id(tags["Event"], event, EVENT)
site_id = get_id(tags["Site"], site, SITE)
date = tags["Date"]
game_round = tags['Round']
white_id = get_id(white, player, PLAYER)
black_id = get_id(black, player, PLAYER)
result = tags["Result"]
if result in pgn2Const:
result = pgn2Const[result]
else:
result = RUNNING
white_elo = tags['WhiteElo']
black_elo = tags['BlackElo']
time_control = tags["TimeControl"]
eco = tags["ECO"][:3]
fen = tags["FEN"]
board_tag = int(tags["Board"]) if "Board" in tags else 0
annotator_id = get_id(tags["Annotator"], annotator,
ANNOTATOR)
source_id = get_id(orig_filename,
source,
SOURCE,
info=info)
ply_count = tags["PlyCount"] if "PlyCount" in tags else 0
offset = base_offset + int(tags["offset"])
self.game_data.append({
'offset': offset,
'offset8': (offset >> 3) << 3,
'event_id': event_id,
'site_id': site_id,
'date': date,
'round': game_round,
'white_id': white_id,
'black_id': black_id,
'result': result,
'white_elo': white_elo,
'black_elo': black_elo,
'ply_count': ply_count,
'eco': eco,
'fen': fen,
'variant': variant,
'board': board_tag,
'time_control': time_control,
'annotator_id': annotator_id,
'source_id': source_id,
})
for tag in tags:
if tag not in dedicated_tags and tag not in other_game_tags and tags[
tag]:
self.tag_game_data.append({
'game_id':
self.next_id[GAME],
'tag_name':
tag,
'tag_value':
tags[tag],
})
self.next_id[GAME] += 1
i += 1
if len(self.game_data) >= self.CHUNK:
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player,
self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator,
self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source,
self.source_data)
self.source_data = []
if self.tag_game_data:
self.conn.execute(self.ins_tag_game,
self.tag_game_data)
self.tag_game_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
i / float(all_games))
GLib.idle_add(
progressbar.set_text,
_("%(counter)s game headers from %(filename)s imported"
% ({
"counter": i,
"filename": basename
})))
else:
log.info("From %s imported %s" % (pgnfile, i))
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source, self.source_data)
self.source_data = []
if self.tag_game_data:
self.conn.execute(self.ins_tag_game, self.tag_game_data)
self.tag_game_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
i / float(all_games))
GLib.idle_add(
progressbar.set_text,
_("%(counter)s game headers from %(filename)s imported"
% ({
"counter": i,
"filename": basename
})))
else:
log.info("From %s imported %s" % (pgnfile, i))
trans.commit()
if self.append_pgn:
# reopen database to write
self.db_handle.close()
with protosave(self.chessfile.path,
self.append_pgn) as self.db_handle:
log.info("Append from %s to %s" %
(pgnfile, self.chessfile.path))
handle.seek(0)
self.db_handle.writelines(handle)
handle.close()
if self.chessfile.scoutfish is not None:
# create new .scout from pgnfile we are importing
from pychess.Savers.pgn import scoutfish_path
args = [
scoutfish_path, "make", pgnfile,
"%s" % base_offset
]
output = subprocess.check_output(
args, stderr=subprocess.STDOUT).decode()
# append it to our existing one
if output.find("Processing...done") > 0:
old_scout = self.chessfile.scoutfish.db
new_scout = os.path.splitext(pgnfile)[0] + '.scout'
with open(old_scout,
"ab") as file1, open(new_scout,
"rb") as file2:
file1.write(file2.read())
self.chessfile.handle = protoopen(self.chessfile.path)
except SQLAlchemyError as e:
trans.rollback()
log.info("Importing %s failed! \n%s" % (pgnfile, e))
def normalize(text):
text = text.splitlines()
text = " ".join(text)
text = text.replace('. ', '. ').replace('. ', '. ')
text = text.replace(' )', ')').replace(' )', ')')
text = text.replace('( ', '(').replace('( ', '(')
text = text.replace(' }', '}').replace(' }', '}')
text = text.replace('{ ', '{').replace('{ ', '{')
return text
filenames = ("atomic", "chess960rwch", "world_matches", "zh")
for filename in filenames:
print("Creating test methods for %s" % filename)
pgnfile = load(protoopen('gamefiles/%s.pgn' % filename))
pgnfile.get_records()
for i, game in enumerate(pgnfile.games):
print("%s/%s" % (i + 1, len(pgnfile.games)))
if i > 100:
break
orig = normalize(pgnfile.get_movetext(game))
model = pgnfile.loadToModel(game)
new = []
walk(model.boards[0].board, new, model)
new = normalize(" ".join(new))
# create test method
test_method = create_test(orig, new)
import unittest
from pychess.Savers.pgn import load, walk, pattern, MOVE
from pychess.System.protoopen import protoopen
file_names = ("atomic", "chess960rwch", "world_matches", "zh")
file_handles = []
for name in file_names:
file_handles.append(protoopen("gamefiles/%s.pgn" % name))
class PgnTestCase(unittest.TestCase):
@classmethod
def tearDownClass(cls):
for handle in file_handles:
handle.close()
def test_movre(self):
"""Testing SAN pattern regexp"""
moves = "e4 fxg7 g8=Q gxh8=N a2+ axb1# c1=Q+ exd8=N# " + \
"0-0-0 O-O-O 0-0 O-O Ka1 Kxf8 Kxd4+ " + \
"Qc3 Rxh8 B1xg7 Nhxg2 Qe4xd5 Rb7+ Bxg4# N8xb2+ Qaxb7# Qd5xe4+"
matches = [m[MOVE - 1] for m in pattern.findall(moves)]
self.assertEqual(' '.join(matches), ' '.join(moves.split()))
def create_test(o, n):
def test_expected(self):
for orig, new in zip(o.split(), n.split()):
# Seems most .PGN unnecessary contains unambiguous notation
# when second move candidate is invalid (leaves king in check)
def normalize(text):
text = text.splitlines()
text = " ".join(text)
text = text.replace('. ', '. ').replace('. ', '. ')
text = text.replace(' )', ')').replace(' )', ')')
text = text.replace('( ', '(').replace('( ', '(')
text = text.replace(' }', '}').replace(' }', '}')
text = text.replace('{ ', '{').replace('{ ', '{')
return text
filenames = ("atomic", "chess960rwch", "world_matches", "zh")
for filename in filenames:
print("Creating test methods for %s" % filename)
pgnfile = load(protoopen('gamefiles/%s.pgn' % filename))
pgnfile.get_records()
for i, game in enumerate(pgnfile.games):
print("%s/%s" % (i + 1, len(pgnfile.games)))
if i > 100:
break
orig = normalize(pgnfile.get_movetext(game))
model = pgnfile.loadToModel(game)
new = []
walk(model.boards[0].board, new, model)
new = normalize(" ".join(new))
# create test method
test_method = create_test(orig, new)
def start_puzzle_from(filename, index=None):
if filename.lower().endswith(".pgn"):
if filename.startswith("lichess_study"):
chessfile = PGNFile(
protoopen(addDataPrefix("learn/puzzles/%s" % filename),
encoding="utf-8"))
else:
chessfile = PGNFile(
protoopen(addDataPrefix("learn/puzzles/%s" % filename)))
chessfile.limit = 1000
chessfile.init_tag_database()
elif filename.lower().endswith(".olv"):
chessfile = OLVFile(
protoopen(addDataPrefix("learn/puzzles/%s" % filename),
encoding="utf-8"))
records, plys = chessfile.get_records()
progress = puzzles_solving_progress.get(filename, [0] * chessfile.count)
if index is None:
index = progress.index(0)
rec = records[index]
timemodel = TimeModel(0, 0)
gamemodel = LearnModel(timemodel)
gamemodel.set_learn_data(PUZZLE, filename, index, len(records))
chessfile.loadToModel(rec, 0, gamemodel)
engine = discoverer.getEngineByName(discoverer.getEngineLearn())
ponder_off = True
color = gamemodel.boards[0].color
w_name = "" if rec["White"] is None else rec["White"]
b_name = "" if rec["Black"] is None else rec["Black"]
player_name = conf.get("firstName", _("You"))
engine_name = discoverer.getName(engine)
if rec["Event"].startswith("Lichess Practice"):
w_name = player_name if color == WHITE else engine_name
b_name = engine_name if color == WHITE else player_name
opp_name = engine_name if rec["Event"].startswith(
"Lichess Practice") else b_name
if color == WHITE:
p0 = (LOCAL, Human, (WHITE, w_name), w_name)
p1 = (ARTIFICIAL, discoverer.initPlayerEngine,
(engine, BLACK, 20, variants[NORMALCHESS], 20, 0, 0,
ponder_off), b_name)
else:
p0 = (ARTIFICIAL, discoverer.initPlayerEngine,
(engine, WHITE, 20, variants[NORMALCHESS], 20, 0, 0,
ponder_off), w_name)
p1 = (LOCAL, Human, (BLACK, b_name), b_name)
def on_game_started(gamemodel, name, color):
perspective.activate_panel("annotationPanel")
asyncio. async (gamemodel.start_analyzer(
HINT, force_engine=discoverer.getEngineLearn()))
gamemodel.players[1 - color].name = name
gamemodel.emit("players_changed")
gamemodel.connect("game_started", on_game_started, opp_name, color)
def goal_checked(gamemodle):
if gamemodel.reason == PRACTICE_GOAL_REACHED:
progress = puzzles_solving_progress[gamemodel.source]
progress[gamemodel.current_index] = 1
puzzles_solving_progress[gamemodel.source] = progress
gamemodel.connect("goal_checked", goal_checked)
gamemodel.variant.need_initial_board = True
gamemodel.status = WAITING_TO_START
perspective = perspective_manager.get_perspective("games")
asyncio. async (perspective.generalStart(gamemodel, p0, p1))
else:
DATA_FILES += [("share/pychess/sounds", glob('sounds/*.ogg'))]
DATA_FILES += [('share/icons/hicolor/24x24/apps', ['pychess.png'])]
DATA_FILES += [('share/gtksourceview-3.0/language-specs',
['gtksourceview-3.0/language-specs/pgn.lang'])]
# Piece sets
DATA_FILES += [("share/pychess/pieces", glob('pieces/*.png'))]
if not isfile(os.path.abspath("learn/puzzles/mate_in_4.sqlite")):
from pychess.Savers.pgn import PGNFile
from pychess.System.protoopen import protoopen
# Lectures, puzzles, lessons
for filename in glob('learn/puzzles/*.pgn'):
chessfile = PGNFile(protoopen(filename))
chessfile.init_tag_database()
for filename in glob('learn/lessons/*.pgn'):
chessfile = PGNFile(protoopen(filename))
chessfile.init_tag_database()
DATA_FILES += [("share/pychess/learn/puzzles", glob('learn/puzzles/*.olv'))]
DATA_FILES += [("share/pychess/learn/puzzles", glob('learn/puzzles/*.pgn'))]
DATA_FILES += [("share/pychess/learn/puzzles", glob('learn/puzzles/*.sqlite'))]
DATA_FILES += [("share/pychess/learn/lessons", glob('learn/lessons/*.pgn'))]
DATA_FILES += [("share/pychess/learn/lessons", glob('learn/lessons/*.sqlite'))]
DATA_FILES += [("share/pychess/learn/lectures", glob('learn/lectures/*.txt'))]
for dir in [d for d in listdir('pieces') if isdir(os.path.join('pieces', d))]:
DATA_FILES += [("share/pychess/pieces/" + dir,
def do_import(self, filename, info=None, progressbar=None):
self.progressbar = progressbar
orig_filename = filename
count_source = self.conn.execute(
self.count_source.where(source.c.name == orig_filename)).scalar()
if count_source > 0:
print("%s is already imported" % filename)
return
# collect new names not in they dict yet
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
self.source_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
self.tag_game_data = []
if filename.startswith("http"):
filename = download_file(filename, progressbar=progressbar)
if filename is None:
return
else:
if not os.path.isfile(filename):
print("Can't open %s" % filename)
return
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
with zipfile.ZipFile(filename, "r") as zf:
path = os.path.dirname(filename)
files = [
os.path.join(path, f) for f in zf.namelist()
if f.lower().endswith(".pgn")
]
zf.extractall(path)
else:
files = [filename]
for pgnfile in files:
base_offset = self.chessfile.size if self.append_pgn else 0
basename = os.path.basename(pgnfile)
if progressbar is not None:
GLib.idle_add(progressbar.set_text,
"Reading %s ..." % basename)
else:
print("Reading %s ..." % pgnfile)
size = os.path.getsize(pgnfile)
handle = protoopen(pgnfile)
# estimated game count
all_games = max(size / 840, 1)
get_id = self.get_id
# use transaction to avoid autocommit slowness
trans = self.conn.begin()
try:
i = 0
for offs, tags in read_games(handle):
if not tags:
print("Empty game #%s" % (i + 1))
continue
if self.cancel:
trans.rollback()
return
fenstr = tags.get("FEN")
variant = tags.get("Variant")
if variant:
if "fischer" in variant.lower() or "960" in variant:
variant = "Fischerandom"
else:
variant = variant.lower().capitalize()
# Fixes for some non statndard Chess960 .pgn
if fenstr and variant == "Fischerandom":
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
if variant not in name2variant:
print("Unknown variant: %s" % variant)
continue
variant = name2variant[variant].variant
if variant == NORMALCHESS:
# lichess uses tag [Variant "Standard"]
variant = 0
else:
variant = 0
white = tags.get('White')
black = tags.get('Black')
event_id = get_id(tags.get('Event'), event, EVENT)
site_id = get_id(tags.get('Site'), site, SITE)
game_date = tags['Date'].strip()
try:
if game_date and '?' not in game_date:
ymd = game_date.split('.')
if len(ymd) == 3:
game_year, game_month, game_day = map(int, ymd)
else:
game_year, game_month, game_day = int(
game_date[:4]), None, None
elif game_date and '?' not in game_date[:4]:
game_year, game_month, game_day = int(
game_date[:4]), None, None
else:
game_year, game_month, game_day = None, None, None
except:
game_year, game_month, game_day = None, None, None
game_round = tags.get('Round')
white_id = get_id(unicode(white), player, PLAYER)
black_id = get_id(unicode(black), player, PLAYER)
result = tags.get("Result")
if result in pgn2Const:
result = pgn2Const[result]
else:
print("Invalid Result tag in game #%s: %s" %
(i + 1, result))
continue
white_elo = tags.get('WhiteElo')
white_elo = int(
white_elo
) if white_elo and white_elo.isdigit() else None
black_elo = tags.get('BlackElo')
black_elo = int(
black_elo
) if black_elo and black_elo.isdigit() else None
time_control = tags.get("TimeControl")
eco = tags.get("ECO")
eco = eco[:3] if eco else None
fen = tags.get("FEN")
board_tag = tags.get("Board")
annotator_id = get_id(tags.get("Annotator"), annotator,
ANNOTATOR)
source_id = get_id(unicode(orig_filename),
source,
SOURCE,
info=info)
self.next_id[GAME] += 1
ply_count = tags.get("PlyCount")
self.game_data.append({
'offset': base_offset + int(offs),
'offset8': (int(offs) >> 3) << 3,
'event_id': event_id,
'site_id': site_id,
'date_year': game_year,
'date_month': game_month,
'date_day': game_day,
'round': game_round,
'white_id': white_id,
'black_id': black_id,
'result': result,
'white_elo': white_elo,
'black_elo': black_elo,
'ply_count': ply_count,
'eco': eco,
'fen': fen,
'variant': variant,
'board': board_tag,
'time_control': time_control,
'annotator_id': annotator_id,
'source_id': source_id,
})
i += 1
if len(self.game_data) >= self.CHUNK:
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player,
self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator,
self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source,
self.source_data)
self.source_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
i / float(all_games))
GLib.idle_add(
progressbar.set_text,
"%s game headers from %s imported" %
(i, basename))
else:
print(pgnfile, i)
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source, self.source_data)
self.source_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
i / float(all_games))
GLib.idle_add(
progressbar.set_text,
"%s game headers from %s imported" % (i, basename))
else:
print(pgnfile, i)
trans.commit()
if self.append_pgn:
# reopen database to write
self.db_handle.close()
self.db_handle = protosave(self.chessfile.path,
self.append_pgn)
print("Append from %s to %s" %
(pgnfile, self.chessfile.path))
handle.seek(0)
for line in handle:
self.db_handle.write(line)
self.db_handle.close()
self.chessfile.handle = protoopen(self.chessfile.path)
except SQLAlchemyError as e:
trans.rollback()
print("Importing %s failed! \n%s" % (pgnfile, e))
def do_import(self, filename, info=None, progressbar=None):
self.progressbar = progressbar
orig_filename = filename
count_source = self.conn.execute(
self.count_source.where(source.c.name == orig_filename)).scalar()
if count_source > 0:
print("%s is already imported" % filename)
return
# collect new names not in they dict yet
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
self.source_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
self.tag_game_data = []
if filename.startswith("http"):
filename = download_file(filename, progressbar=progressbar)
if filename is None:
return
else:
if not os.path.isfile(filename):
print("Can't open %s" % filename)
return
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
with zipfile.ZipFile(filename, "r") as zf:
path = os.path.dirname(filename)
files = [
os.path.join(path, f) for f in zf.namelist()
if f.lower().endswith(".pgn")
]
zf.extractall(path)
else:
files = [filename]
for pgnfile in files:
base_offset = self.chessfile.size if self.append_pgn else 0
basename = os.path.basename(pgnfile)
if progressbar is not None:
GLib.idle_add(progressbar.set_text,
"Reading %s ..." % basename)
else:
print("Reading %s ..." % pgnfile)
size = os.path.getsize(pgnfile)
handle = protoopen(pgnfile)
# estimated game count
all_games = max(size / 840, 1)
handle_json = None
if pgnextractor is not None:
try:
headers_json = os.path.splitext(
pgnfile)[0] + ".headers.json"
if not os.path.isfile(headers_json):
output = subprocess.check_output(
[pgnextractor, "headers", pgnfile]).decode()
for line in output:
if line.startswith("Games"):
all_games = line.split()[1]
handle_json = protoopen(headers_json)
except subprocess.CalledProcessError:
print("pgnextractor failed")
get_id = self.get_id
# use transaction to avoid autocommit slowness
trans = self.conn.begin()
try:
i = 0
for tags in read_games(handle, handle_json):
if not tags:
print("Empty game #%s" % (i + 1))
continue
if self.cancel:
trans.rollback()
return
fenstr = tags["FEN"] if "FEN" in tags else ""
variant = tags["Variant"] if "Variant" in tags else ""
if variant:
if "fischer" in variant.lower() or "960" in variant:
variant = "Fischerandom"
else:
variant = variant.lower().capitalize()
# Fixes for some non statndard Chess960 .pgn
if fenstr and variant == "Fischerandom":
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
if variant not in name2variant:
print("Unknown variant: %s" % variant)
continue
variant = name2variant[variant].variant
if variant == NORMALCHESS:
# lichess uses tag [Variant "Standard"]
variant = 0
else:
variant = 0
white = tags["White"] if "White" in tags else ""
black = tags["Black"] if "Black" in tags else ""
event_id = get_id(tags["Event"] if "Event" in tags else "",
event, EVENT)
site_id = get_id(tags["Site"] if "Site" in tags else "",
site, SITE)
game_date = tags["Date"] if "Date" in tags else ""
try:
if game_date and '?' not in game_date:
ymd = game_date.split('.')
if len(ymd) == 3:
game_year, game_month, game_day = map(int, ymd)
else:
game_year, game_month, game_day = int(
game_date[:4]), None, None
elif game_date and '?' not in game_date[:4]:
game_year, game_month, game_day = int(
game_date[:4]), None, None
else:
game_year, game_month, game_day = None, None, None
except:
game_year, game_month, game_day = None, None, None
game_round = tags['Round'] if "Round" in tags else ""
white_id = get_id(white, player, PLAYER)
black_id = get_id(black, player, PLAYER)
result = tags["Result"] if "Result" in tags else ""
if result in pgn2Const:
result = pgn2Const[result]
else:
if basename != "eco.pgn":
print("Invalid Result tag in game #%s: %s" %
(i + 1, result))
continue
white_elo = tags['WhiteElo'] if "WhiteElo" in tags else "0"
white_elo = int(
white_elo) if white_elo and white_elo.isdigit() else 0
black_elo = tags['BlackElo'] if "BlackElo" in tags else ""
black_elo = int(
black_elo) if black_elo and black_elo.isdigit() else 0
time_control = tags[
"TimeControl"] if "TimeControl" in tags else ""
eco = tags["ECO"][:3] if "ECO" in tags else ""
fen = tags["FEN"] if "FEN" in tags else ""
board_tag = int(tags["Board"]) if "Board" in tags else 0
annotator_id = get_id(
tags["Annotator"] if "Annotator" in tags else "",
annotator, ANNOTATOR)
source_id = get_id(orig_filename,
source,
SOURCE,
info=info)
self.next_id[GAME] += 1
ply_count = tags["PlyCount"] if "PlyCount" in tags else 0
offset = base_offset + int(tags["offset"])
self.game_data.append({
'offset': offset,
'offset8': (offset >> 3) << 3,
'event_id': event_id,
'site_id': site_id,
'date_year': game_year,
'date_month': game_month,
'date_day': game_day,
'round': game_round,
'white_id': white_id,
'black_id': black_id,
'result': result,
'white_elo': white_elo,
'black_elo': black_elo,
'ply_count': ply_count,
'eco': eco,
'fen': fen,
'variant': variant,
'board': board_tag,
'time_control': time_control,
'annotator_id': annotator_id,
'source_id': source_id,
})
i += 1
if len(self.game_data) >= self.CHUNK:
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player,
self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator,
self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source,
self.source_data)
self.source_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
i / float(all_games))
GLib.idle_add(
progressbar.set_text,
"%s game headers from %s imported" %
(i, basename))
else:
print(pgnfile, i)
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source, self.source_data)
self.source_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
i / float(all_games))
GLib.idle_add(
progressbar.set_text,
"%s game headers from %s imported" % (i, basename))
else:
print(pgnfile, i)
trans.commit()
if self.append_pgn:
# reopen database to write
self.db_handle.close()
self.db_handle = protosave(self.chessfile.path,
self.append_pgn)
print("Append from %s to %s" %
(pgnfile, self.chessfile.path))
handle.seek(0)
self.db_handle.writelines(handle)
self.db_handle.close()
handle.close()
if self.chessfile.scoutfish is not None:
# create new .scout from pgnfile we are importing
from pychess.Savers.pgn import scoutfish_path
args = [
scoutfish_path, "make", pgnfile,
"%s" % base_offset
]
output = subprocess.check_output(
args, stderr=subprocess.STDOUT).decode()
# append it to our existing one
if output.find("Processing...done") > 0:
old_scout = self.chessfile.scoutfish.db
new_scout = os.path.splitext(pgnfile)[0] + '.scout'
with open(old_scout,
"ab") as file1, open(new_scout,
"rb") as file2:
file1.write(file2.read())
self.chessfile.handle = protoopen(self.chessfile.path)
except SQLAlchemyError as e:
trans.rollback()
print("Importing %s failed! \n%s" % (pgnfile, e))
def do_import(self, filename, info=None, progressbar=None):
self.progressbar = progressbar
orig_filename = filename
count_source = self.conn.execute(self.count_source.where(source.c.name == orig_filename)).scalar()
if count_source > 0:
print("%s is already imported" % filename)
return
# collect new names not in they dict yet
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
self.source_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
self.tag_game_data = []
if filename.startswith("http"):
filename = download_file(filename, progressbar=progressbar)
if filename is None:
return
else:
if not os.path.isfile(filename):
print("Can't open %s" % filename)
return
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
with zipfile.ZipFile(filename, "r") as zf:
path = os.path.dirname(filename)
files = [os.path.join(path, f) for f in zf.namelist() if f.lower().endswith(".pgn")]
zf.extractall(path)
else:
files = [filename]
for pgnfile in files:
base_offset = self.chessfile.size if self.append_pgn else 0
basename = os.path.basename(pgnfile)
if progressbar is not None:
GLib.idle_add(progressbar.set_text, "Reading %s ..." % basename)
else:
print("Reading %s ..." % pgnfile)
size = os.path.getsize(pgnfile)
handle = protoopen(pgnfile)
# estimated game count
all_games = max(size / 840, 1)
handle_json = None
if pgnextractor is not None:
try:
headers_json = os.path.splitext(pgnfile)[0] + ".headers.json"
if not os.path.isfile(headers_json):
output = subprocess.check_output([pgnextractor, "headers", pgnfile]).decode()
for line in output:
if line.startswith("Games"):
all_games = line.split()[1]
handle_json = protoopen(headers_json)
except subprocess.CalledProcessError:
print("pgnextractor failed")
get_id = self.get_id
# use transaction to avoid autocommit slowness
trans = self.conn.begin()
try:
i = 0
for tags in read_games(handle, handle_json):
if not tags:
print("Empty game #%s" % (i + 1))
continue
if self.cancel:
trans.rollback()
return
fenstr = tags["FEN"] if "FEN" in tags else ""
variant = tags["Variant"] if "Variant" in tags else ""
if variant:
if "fischer" in variant.lower() or "960" in variant:
variant = "Fischerandom"
else:
variant = variant.lower().capitalize()
# Fixes for some non statndard Chess960 .pgn
if fenstr and variant == "Fischerandom":
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
if variant not in name2variant:
print("Unknown variant: %s" % variant)
continue
variant = name2variant[variant].variant
if variant == NORMALCHESS:
# lichess uses tag [Variant "Standard"]
variant = 0
else:
variant = 0
white = tags["White"] if "White" in tags else ""
black = tags["Black"] if "Black" in tags else ""
event_id = get_id(tags["Event"] if "Event" in tags else "", event, EVENT)
site_id = get_id(tags["Site"] if "Site" in tags else "", site, SITE)
game_date = tags["Date"] if "Date" in tags else ""
try:
if game_date and '?' not in game_date:
ymd = game_date.split('.')
if len(ymd) == 3:
game_year, game_month, game_day = map(int, ymd)
else:
game_year, game_month, game_day = int(game_date[:4]), None, None
elif game_date and '?' not in game_date[:4]:
game_year, game_month, game_day = int(game_date[:4]), None, None
else:
game_year, game_month, game_day = None, None, None
except:
game_year, game_month, game_day = None, None, None
game_round = tags['Round'] if "Round" in tags else ""
white_id = get_id(white, player, PLAYER)
black_id = get_id(black, player, PLAYER)
result = tags["Result"] if "Result" in tags else ""
if result in pgn2Const:
result = pgn2Const[result]
else:
if basename != "eco.pgn":
print("Invalid Result tag in game #%s: %s" % (i + 1, result))
continue
white_elo = tags['WhiteElo'] if "WhiteElo" in tags else "0"
white_elo = int(white_elo) if white_elo and white_elo.isdigit() else 0
black_elo = tags['BlackElo'] if "BlackElo" in tags else ""
black_elo = int(black_elo) if black_elo and black_elo.isdigit() else 0
time_control = tags["TimeControl"] if "TimeControl" in tags else ""
eco = tags["ECO"][:3] if "ECO" in tags else ""
fen = tags["FEN"] if "FEN" in tags else ""
board_tag = int(tags["Board"]) if "Board" in tags else 0
annotator_id = get_id(tags["Annotator"] if "Annotator" in tags else "", annotator, ANNOTATOR)
source_id = get_id(orig_filename, source, SOURCE, info=info)
self.next_id[GAME] += 1
ply_count = tags["PlyCount"] if "PlyCount" in tags else 0
offset = base_offset + int(tags["offset"])
self.game_data.append({
'offset': offset,
'offset8': (offset >> 3) << 3,
'event_id': event_id,
'site_id': site_id,
'date_year': game_year,
'date_month': game_month,
'date_day': game_day,
'round': game_round,
'white_id': white_id,
'black_id': black_id,
'result': result,
'white_elo': white_elo,
'black_elo': black_elo,
'ply_count': ply_count,
'eco': eco,
'fen': fen,
'variant': variant,
'board': board_tag,
'time_control': time_control,
'annotator_id': annotator_id,
'source_id': source_id,
})
i += 1
if len(self.game_data) >= self.CHUNK:
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player,
self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator,
self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source, self.source_data)
self.source_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction, i / float(all_games))
GLib.idle_add(progressbar.set_text, "%s game headers from %s imported" % (i, basename))
else:
print(pgnfile, i)
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source, self.source_data)
self.source_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction, i / float(all_games))
GLib.idle_add(progressbar.set_text, "%s game headers from %s imported" % (i, basename))
else:
print(pgnfile, i)
trans.commit()
if self.append_pgn:
# reopen database to write
self.db_handle.close()
self.db_handle = protosave(self.chessfile.path, self.append_pgn)
print("Append from %s to %s" % (pgnfile, self.chessfile.path))
handle.seek(0)
self.db_handle.writelines(handle)
self.db_handle.close()
handle.close()
if self.chessfile.scoutfish is not None:
# create new .scout from pgnfile we are importing
from pychess.Savers.pgn import scoutfish_path
args = [scoutfish_path, "make", pgnfile, "%s" % base_offset]
output = subprocess.check_output(args, stderr=subprocess.STDOUT).decode()
# append it to our existing one
if output.find("Processing...done") > 0:
old_scout = self.chessfile.scoutfish.db
new_scout = os.path.splitext(pgnfile)[0] + '.scout'
with open(old_scout, "ab") as file1, open(new_scout, "rb") as file2:
file1.write(file2.read())
self.chessfile.handle = protoopen(self.chessfile.path)
except SQLAlchemyError as e:
trans.rollback()
print("Importing %s failed! \n%s" % (pgnfile, e))
