def print_db(self):
a1 = event.alias()
a2 = site.alias()
a3 = player.alias()
a4 = player.alias()
s = select([
game.c.id,
a1.c.name.label('event'),
a2.c.name.label('site'),
a3.c.name.label('white'),
a4.c.name.label('black'), game.c.date_year, game.c.date_month,
game.c.date_day, game.c.eco, game.c.result, game.c.white_elo,
game.c.black_elo
],
and_(game.c.event_id == a1.c.id, game.c.site_id == a2.c.id,
game.c.white_id == a3.c.id,
game.c.black_id == a4.c.id)).where(
and_(a3.c.name.startswith(unicode("Réti")),
a4.c.name.startswith(unicode("Van Nüss"))))
result = self.conn.execute(s)
games = result.fetchall()
for g in games:
print("%s %s %s %s %s %s %s %s %s %s %s %s" %
(g['id'], g['event'], g['site'], g['white'], g['black'],
g[5], g[6], g[7], g['eco'], reprResult[g['result']],
g['white_elo'], g['black_elo']))
def ini_tag(engine):
conn = engine.connect()
new_values = [
{"id": LOCAL, "name": unicode("Local game")},
{"id": ARTIFICIAL, "name": unicode("Chess engine(s)")},
{"id": REMOTE, "name": unicode("ICS game")},
]
conn.execute(tag.insert(), new_values)
conn.close()
def ini_collection():
conn = engine.connect()
new_values = [
{"id": LOCAL, "name": unicode("Local game")},
{"id": ARTIFICIAL, "name": unicode("Chess engine(s)")},
{"id": REMOTE, "name": unicode("ICS game")},
]
conn.execute(collection.insert(), new_values)
conn.close()
def __movestr(self, board):
move = board.lastMove
if self.fan:
movestr = unicode(toFAN(board.prev, move))
else:
movestr = unicode(toSAN(board.prev, move, True))
nagsymbols = "".join([nag2symbol(nag) for nag in board.nags])
# To prevent wrap castling we will use hyphen bullet (U+2043)
return "%s%s%s" % (move_count(board), movestr.replace(u'-', u'\u2043'), nagsymbols)
def get_id(table, name):
if not name:
return None
selection = select([table.c.id], table.c.name == unicode(name))
result = conn.execute(selection)
id_ = result.scalar()
if id_ is None:
result = conn.execute(table.insert().values(name=unicode(name)))
id_ = result.inserted_primary_key[0]
return id_
def get_rating_range_display_text (rmin=0, rmax=9999):
assert isinstance(rmin, type(int())) and rmin >= 0 and rmin <= 9999, rmin
assert isinstance(rmax, type(int())) and rmax >= 0 and rmax <= 9999, rmax
if rmin > 0:
text = "%d" % rmin
if rmax == 9999:
text += unicode("↑")
else:
text += "-%d" % rmax
elif rmax != 9999:
text = unicode("%d↓" % rmax)
else:
text = None
return text
def get_rating_range_display_text(rmin=0, rmax=9999):
assert isinstance(rmin, type(int())) and rmin >= 0 and rmin <= 9999, rmin
assert isinstance(rmax, type(int())) and rmax >= 0 and rmax <= 9999, rmax
if rmin > 0:
text = "%d" % rmin
if rmax == 9999:
text += unicode("↑")
else:
text += "-%d" % rmax
elif rmax != 9999:
text = unicode("%d↓" % rmax)
else:
text = None
return text
def variation_end(self, iter, index, level, firstboard, parent, opening_node):
start = iter.get_offset()
if level == 0:
self.textbuffer.insert_with_tags_by_name(iter, "]", "variation-toplevel", "variation-margin0")
elif (level+1) % 2 == 0:
self.textbuffer.insert_with_tags_by_name(iter, ")", "variation-even", "variation-margin1")
else:
self.textbuffer.insert_with_tags_by_name(iter, ")", "variation-uneven", "variation-margin2")
self.textbuffer.insert_with_tags_by_name(iter, unicode("✖ "), "remove-variation")
chr = iter.get_char()
# somehow iter.begins_tag() doesn't work, so we use get_char() instead
if iter.get_char() != "\n":
self.textbuffer.insert_with_tags_by_name(iter, "\n", "new_line")
node = {}
node["board"] = firstboard
node["parent"] = parent
node["vari"] = opening_node
node["start"] = start
node["end"] = iter.get_offset()
if index == -1:
self.nodelist.append(node)
else:
self.nodelist.insert(index, node)
return iter.get_offset() - start
def variation_end(self, iter, index, level, firstboard, parent, opening_node):
start = iter.get_offset()
if level == 0:
self.textbuffer.insert_with_tags_by_name(iter, "]", "variation-toplevel", "variation-margin0")
elif (level+1) % 2 == 0:
self.textbuffer.insert_with_tags_by_name(iter, ")", "variation-even", "variation-margin1")
else:
self.textbuffer.insert_with_tags_by_name(iter, ")", "variation-uneven", "variation-margin2")
self.textbuffer.insert_with_tags_by_name(iter, unicode("✖ "), "remove-variation")
chr = iter.get_char()
# somehow iter.begins_tag() doesn't work, so we use get_char() instead
if iter.get_char() != "\n":
self.textbuffer.insert_with_tags_by_name(iter, "\n", "new_line")
node = {}
node["board"] = firstboard
node["parent"] = parent
node["vari"] = opening_node
node["start"] = start
node["end"] = iter.get_offset()
if index == -1:
self.nodelist.append(node)
else:
self.nodelist.insert(index, node)
return iter.get_offset() - start
def importing():
drop_indexes(self.gamelist.chessfile.engine)
self.importer = PgnImport(self.gamelist.chessfile.engine)
for i, filename in enumerate(filenames):
filename = unicode(filename)
GLib.idle_add(self.progressbar0.set_fraction, i / float(len(filenames)))
# GLib.idle_add(self.progressbar0.set_text, filename)
if self.importer.cancel:
break
if isinstance(filename, tuple):
info_link, pgn_link = filename
self.importer.do_import(pgn_link, info=info_link, progressbar=self.progressbar)
else:
self.importer.do_import(filename, progressbar=self.progressbar)
GLib.idle_add(self.progressbar.set_text, "Recreating indexes...")
create_indexes(self.gamelist.chessfile.engine)
self.gamelist.offset = 0
self.gamelist.chessfile.build_where_tags("")
self.gamelist.chessfile.build_where_bitboards(0, 0)
self.gamelist.chessfile.build_query()
self.gamelist.chessfile.update_count()
self.gamelist.chessfile.update_count_stats()
GLib.idle_add(self.gamelist.load_games)
GLib.idle_add(self.emit, "chessfile_imported", self.gamelist.chessfile)
GLib.idle_add(self.progress_dialog.hide)
def toFAN(board, move):
""" Returns a Figurine Algebraic Notation string of a move """
san = unicode(toSAN(board, move))
if board.color == WHITE:
return san.translate(san2WhiteFanDic)
else:
return san.translate(san2BlackFanDic)
def toFAN (board, move):
""" Returns a Figurine Algebraic Notation string of a move """
san = unicode(toSAN (board, move))
if board.color == WHITE:
return san.translate(san2WhiteFanDic)
else:
return san.translate(san2BlackFanDic)
def feed(pgnfile, lang):
cf = load(protoopen(pgnfile))
rows = []
old_eco = ""
ply_max = 0
for i, game in enumerate(cf.games):
model = cf.loadToModel(i)
eco = cf._getTag(i, "ECO")[:3]
opening = cf._getTag(i, "Opening")
if opening is None:
opening = ""
variation = cf._getTag(i, "Variation")
if variation is None:
variation = ""
base = int(old_eco != eco)
ply = len(model.moves)
ply_max = max(ply_max, ply)
if ply == 0:
cu = conn.cursor()
cu.execute(
"select * from openings where eco=? and lang='en' and base=1",
(eco, ))
res = cu.fetchone()
if res is not None:
hash = res[0]
else:
hash = memoryview(hash_struct.pack(
model.boards[-1].board.hash))
if opening:
rows.append((hash, base, unicode(eco), unicode(lang),
unicode(opening), unicode(variation)))
old_eco = eco
c.executemany(
"insert into openings(hash, base, eco, lang, opening, variation) values (?, ?, ?, ?, ?, ?)",
rows)
conn.commit()
print("Max ply was %s" % ply_max)
def ellipsize(string, maxlen):
""" Description: given a string and a length ellipsize will return the string
if it is smaller than length or it will return the string truncated to length
and append ... to it
Return type : str
"""
if len(string) <= maxlen or maxlen < 4:
return string
return string[:maxlen - 1] + unicode("…")
def ellipsize(string, maxlen):
""" Description: given a string and a length ellipsize will return the string
if it is smaller than length or it will return the string truncated to length
and append ... to it
Return type : str
"""
if len(string) <= maxlen or maxlen < 4:
return string
return string[:maxlen - 1] + unicode("…")
def build_where_tags(self, text):
if text:
text = unicode(text)
self.where_tags = or_(
pl1.c.name.startswith(text),
pl2.c.name.startswith(text),
event.c.name.startswith(text),
site.c.name.startswith(text),
annotator.c.name.startswith(text),
)
else:
self.where_tags = None
def feed(pgnfile, lang):
cf = load(protoopen(pgnfile))
rows = []
old_eco = ""
ply_max = 0
for i, game in enumerate(cf.games):
model = cf.loadToModel(i)
eco = cf._getTag(i, "ECO")[:3]
opening = cf._getTag(i, "Opening")
if opening is None:
opening = ""
variation = cf._getTag(i, "Variation")
if variation is None:
variation = ""
base = int(old_eco != eco)
ply = len(model.moves)
ply_max = max(ply_max, ply)
if ply == 0:
cu = conn.cursor()
cu.execute("select * from openings where eco=? and lang='en' and base=1", (eco,))
res = cu.fetchone()
if res is not None:
hash = res[0]
else:
hash = memoryview(hash_struct.pack(model.boards[-1].board.hash))
if opening:
rows.append((hash, base, unicode(eco), unicode(lang), unicode(opening), unicode(variation)))
old_eco = eco
c.executemany("insert into openings(hash, base, eco, lang, opening, variation) values (?, ?, ?, ?, ?, ?)", rows)
conn.commit()
print("Max ply was %s" % ply_max)
def newMessage (cls, tag, timestamp, message, importance):
textview = cls._getPageFromTag(tag)["textview"]
if not tag in cls.tagToTime or timestamp-cls.tagToTime[tag] >= 1:
t = time.strftime("%H:%M:%S", time.localtime(timestamp))
textview.get_buffer().insert_with_tags_by_name(
textview.get_buffer().get_end_iter(), "\n%s\n%s\n"%(t,"-"*60), str(logging.INFO))
cls.tagToTime[tag] = timestamp
if isinstance(message, str):
message = unicode(message, "utf-8", 'rawreplace')
if not message.endswith("\n"):
message = "%s\n" % message
textview.get_buffer().insert_with_tags_by_name(
textview.get_buffer().get_end_iter(), message, str(importance))
def print_db(self):
a1 = event.alias()
a2 = site.alias()
a3 = player.alias()
a4 = player.alias()
s = select(
[game.c.id, a1.c.name.label('event'), a2.c.name.label('site'),
a3.c.name.label('white'), a4.c.name.label('black'),
game.c.date_year, game.c.date_month, game.c.date_day, game.c.eco,
game.c.result, game.c.white_elo, game.c.black_elo],
and_(game.c.event_id == a1.c.id, game.c.site_id == a2.c.id,
game.c.white_id == a3.c.id,
game.c.black_id == a4.c.id)).where(and_(
a3.c.name.startswith(unicode(
"Réti")), a4.c.name.startswith(unicode("Van Nüss"))))
result = self.conn.execute(s)
games = result.fetchall()
for g in games:
print("%s %s %s %s %s %s %s %s %s %s %s %s" %
(g['id'], g['event'], g['site'], g['white'], g['black'],
g[5], g[6], g[7], g['eco'], reprResult[g['result']],
g['white_elo'], g['black_elo']))
def newMessage(cls, tag, timestamp, message, importance):
textview = cls._getPageFromTag(tag)["textview"]
if not tag in cls.tagToTime or timestamp - cls.tagToTime[tag] >= 1:
t = time.strftime("%H:%M:%S", time.localtime(timestamp))
textview.get_buffer().insert_with_tags_by_name(
textview.get_buffer().get_end_iter(),
"\n%s\n%s\n" % (t, "-" * 60), str(logging.INFO))
cls.tagToTime[tag] = timestamp
if isinstance(message, str):
message = unicode(message, "utf-8", 'rawreplace')
if not message.endswith("\n"):
message = "%s\n" % message
textview.get_buffer().insert_with_tags_by_name(
textview.get_buffer().get_end_iter(), message, str(importance))
def importing():
drop_indexes(self.gamelist.chessfile.engine)
self.importer = PgnImport(self.gamelist.chessfile, append_pgn=True)
for i, filename in enumerate(filenames):
GLib.idle_add(self.progressbar0.set_fraction, i / float(len(filenames)))
# GLib.idle_add(self.progressbar0.set_text, filename)
if self.importer.cancel:
break
if isinstance(filename, tuple):
info_link, pgn_link = filename
self.importer.do_import(pgn_link, info=info_link, progressbar=self.progressbar1)
else:
filename = unicode(filename)
self.importer.do_import(filename, progressbar=self.progressbar1)
GLib.idle_add(self.progressbar1.set_text, "Recreating indexes...")
# .sqlite
create_indexes(self.gamelist.chessfile.engine)
# .scout
self.gamelist.chessfile.init_scoutfish()
if self.gamelist.chessfile.scoutfish is not None:
self.gamelist.chessfile.scoutfish.make()
# .bin
self.gamelist.chessfile.init_chess_db()
if self.gamelist.chessfile.chess_db is not None:
self.gamelist.chessfile.chess_db.make()
self.gamelist.chessfile.set_tags_filter("")
self.gamelist.chessfile.set_fen_filter(FEN_START)
self.gamelist.chessfile.set_scout_filter("")
GLib.idle_add(self.gamelist.load_games)
GLib.idle_add(self.emit, "chessfile_imported", self.gamelist.chessfile)
GLib.idle_add(self.progress_dialog.hide)
def do_import(self, filename, info=None, progressbar=None):
DB_MAXINT_SHIFT = get_maxint_shift(self.engine)
self.progressbar = progressbar
orig_filename = filename
count_source = self.conn.execute(
self.count_source.where(source.c.name == orig_filename)).scalar()
if count_source > 0:
print("%s is already imported" % filename)
return
# collect new names not in they dict yet
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
self.source_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
self.bitboard_data = []
self.stat_ins_data = []
self.stat_upd_data = []
self.tag_game_data = []
if filename.startswith("http"):
filename = download_file(filename, progressbar=progressbar)
if filename is None:
return
else:
if not os.path.isfile(filename):
print("Can't open %s" % filename)
return
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
zf = zipfile.ZipFile(filename, "r")
files = [f for f in zf.namelist() if f.lower().endswith(".pgn")]
else:
zf = None
files = [filename]
for pgnfile in files:
basename = os.path.basename(pgnfile)
if progressbar is not None:
GLib.idle_add(progressbar.set_text,
"Reading %s ..." % basename)
else:
print("Reading %s ..." % pgnfile)
if zf is None:
size = os.path.getsize(pgnfile)
handle = protoopen(pgnfile)
else:
size = zf.getinfo(pgnfile).file_size
handle = io.TextIOWrapper(zf.open(pgnfile),
encoding=PGN_ENCODING,
newline='')
cf = PgnBase(handle, [])
# estimated game count
all_games = max(size / 840, 1)
self.CHUNK = 1000 if all_games > 5000 else 100
get_id = self.get_id
# use transaction to avoid autocommit slowness
trans = self.conn.begin()
try:
i = 0
for tagtext, movetext in read_games(handle):
tags = defaultdict(str, tagre.findall(tagtext))
if not tags:
print("Empty game #%s" % (i + 1))
continue
if self.cancel:
trans.rollback()
return
fenstr = tags.get("FEN")
variant = tags.get("Variant")
if variant:
if "fischer" in variant.lower() or "960" in variant:
variant = "Fischerandom"
else:
variant = variant.lower().capitalize()
# Fixes for some non statndard Chess960 .pgn
if fenstr and variant == "Fischerandom":
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
if variant not in name2variant:
print("Unknown variant: %s" % variant)
continue
variant = name2variant[variant].variant
if variant == NORMALCHESS:
# lichess uses tag [Variant "Standard"]
variant = 0
board = START_BOARD.clone()
else:
board = LBoard(variant)
elif fenstr:
variant = 0
board = LBoard()
else:
variant = 0
board = START_BOARD.clone()
if fenstr:
try:
board.applyFen(fenstr)
except SyntaxError as e:
print(
_("The game #%s can't be loaded, because of an error parsing FEN"
) % (i + 1), e.args[0])
continue
elif variant:
board.applyFen(FEN_START)
movelist = array("H")
comments = []
cf.error = None
# First we try to use simple_parse_movetext()
# assuming most games in .pgn contains only moves
# without any comments/variations
simple = False
if not fenstr and not variant:
bitboards = []
simple = cf.simple_parse_movetext(
movetext, board, movelist, bitboards)
if cf.error is not None:
print("ERROR in %s game #%s" % (pgnfile, i + 1),
cf.error.args[0])
continue
# If simple_parse_movetext() find any comments/variations
# we restart parsing with full featured parse_movetext()
if not simple:
movelist = array("H")
bitboards = None
# in case simple_parse_movetext failed we have to reset our lboard
if not fenstr and not variant:
board = START_BOARD.clone()
# parse movetext to create boards tree structure
boards = [board]
boards = cf.parse_movetext(movetext,
boards[0],
-1,
pgn_import=True)
if cf.error is not None:
print("ERROR in %s game #%s" % (pgnfile, i + 1),
cf.error.args[0])
continue
# create movelist and comments from boards tree
walk(boards[0], movelist, comments)
white = tags.get('White')
black = tags.get('Black')
if not movelist:
if (not comments) and (not white) and (not black):
print("Empty game #%s" % (i + 1))
continue
event_id = get_id(tags.get('Event'), event, EVENT)
site_id = get_id(tags.get('Site'), site, SITE)
game_date = tags.get('Date').strip()
try:
if game_date and '?' not in game_date:
ymd = game_date.split('.')
if len(ymd) == 3:
game_year, game_month, game_day = map(int, ymd)
else:
game_year, game_month, game_day = int(
game_date[:4]), None, None
elif game_date and '?' not in game_date[:4]:
game_year, game_month, game_day = int(
game_date[:4]), None, None
else:
game_year, game_month, game_day = None, None, None
except:
game_year, game_month, game_day = None, None, None
game_round = tags.get('Round')
white_fide_id = tags.get('WhiteFideId')
black_fide_id = tags.get('BlackFideId')
white_id = get_id(unicode(white),
player,
PLAYER,
fide_id=white_fide_id)
black_id = get_id(unicode(black),
player,
PLAYER,
fide_id=black_fide_id)
result = tags.get("Result")
if result in pgn2Const:
result = pgn2Const[result]
else:
print("Invalid Result tag in game #%s: %s" %
(i + 1, result))
continue
white_elo = tags.get('WhiteElo')
white_elo = int(
white_elo
) if white_elo and white_elo.isdigit() else None
black_elo = tags.get('BlackElo')
black_elo = int(
black_elo
) if black_elo and black_elo.isdigit() else None
time_control = tags.get("TimeControl")
eco = tags.get("ECO")
eco = eco[:3] if eco else None
fen = tags.get("FEN")
board_tag = tags.get("Board")
annotator_id = get_id(tags.get("Annotator"), annotator,
ANNOTATOR)
source_id = get_id(unicode(orig_filename),
source,
SOURCE,
info=info)
game_id = self.next_id[GAME]
self.next_id[GAME] += 1
# annotated game
if bitboards is None:
for ply, board in enumerate(boards):
if ply == 0:
continue
bb = board.friends[0] | board.friends[1]
# Avoid to include mate in x .pgn collections and similar in opening tree
if fen and "/pppppppp/8/8/8/8/PPPPPPPP/" not in fen:
ply = -1
self.bitboard_data.append({
'game_id':
game_id,
'ply':
ply,
'bitboard':
bb - DB_MAXINT_SHIFT,
})
if ply <= STAT_PLY_MAX:
self.stat_ins_data.append({
'ply': ply,
'bitboard': bb - DB_MAXINT_SHIFT,
'count': 0,
'whitewon': 0,
'blackwon': 0,
'draw': 0,
'white_elo_count': 0,
'black_elo_count': 0,
'white_elo': 0,
'black_elo': 0,
})
self.stat_upd_data.append({
'_ply':
ply,
'_bitboard':
bb - DB_MAXINT_SHIFT,
'_count':
1,
'_whitewon':
1 if result == WHITEWON else 0,
'_blackwon':
1 if result == BLACKWON else 0,
'_draw':
1 if result == DRAW else 0,
'_white_elo_count':
1 if white_elo is not None else 0,
'_black_elo_count':
1 if black_elo is not None else 0,
'_white_elo':
white_elo if white_elo is not None else 0,
'_black_elo':
black_elo if black_elo is not None else 0,
})
# simple game
else:
for ply, bb in enumerate(bitboards):
if ply == 0:
continue
self.bitboard_data.append({
'game_id':
game_id,
'ply':
ply,
'bitboard':
bb - DB_MAXINT_SHIFT,
})
if ply <= STAT_PLY_MAX:
self.stat_ins_data.append({
'ply': ply,
'bitboard': bb - DB_MAXINT_SHIFT,
'count': 0,
'whitewon': 0,
'blackwon': 0,
'draw': 0,
'white_elo_count': 0,
'black_elo_count': 0,
'white_elo': 0,
'black_elo': 0,
})
self.stat_upd_data.append({
'_ply':
ply,
'_bitboard':
bb - DB_MAXINT_SHIFT,
'_count':
1,
'_whitewon':
1 if result == WHITEWON else 0,
'_blackwon':
1 if result == BLACKWON else 0,
'_draw':
1 if result == DRAW else 0,
'_white_elo_count':
1 if white_elo is not None else 0,
'_black_elo_count':
1 if black_elo is not None else 0,
'_white_elo':
white_elo if white_elo is not None else 0,
'_black_elo':
black_elo if black_elo is not None else 0,
})
ply_count = tags.get("PlyCount")
if not ply_count and not fen:
ply_count = len(
bitboards) if bitboards is not None else len(
boards)
self.game_data.append({
'event_id':
event_id,
'site_id':
site_id,
'date_year':
game_year,
'date_month':
game_month,
'date_day':
game_day,
'round':
game_round,
'white_id':
white_id,
'black_id':
black_id,
'result':
result,
'white_elo':
white_elo,
'black_elo':
black_elo,
'ply_count':
ply_count,
'eco':
eco,
'fen':
fen,
'variant':
variant,
'board':
board_tag,
'time_control':
time_control,
'annotator_id':
annotator_id,
'source_id':
source_id,
'movelist':
movelist.tostring(),
'comments':
unicode("|".join(comments)),
})
i += 1
if len(self.game_data) >= self.CHUNK:
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player,
self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator,
self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source,
self.source_data)
self.source_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if self.bitboard_data:
self.conn.execute(self.ins_bitboard,
self.bitboard_data)
self.bitboard_data = []
self.conn.execute(self.ins_stat,
self.stat_ins_data)
self.conn.execute(self.upd_stat,
self.stat_upd_data)
self.stat_ins_data = []
self.stat_upd_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
i / float(all_games))
GLib.idle_add(
progressbar.set_text,
"%s games from %s imported" % (i, basename))
else:
print(pgnfile, i)
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source, self.source_data)
self.source_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if self.bitboard_data:
self.conn.execute(self.ins_bitboard, self.bitboard_data)
self.bitboard_data = []
self.conn.execute(self.ins_stat, self.stat_ins_data)
self.conn.execute(self.upd_stat, self.stat_upd_data)
self.stat_ins_data = []
self.stat_upd_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
i / float(all_games))
GLib.idle_add(progressbar.set_text,
"%s games from %s imported" % (i, basename))
else:
print(pgnfile, i)
trans.commit()
except SQLAlchemyError as e:
trans.rollback()
print("Importing %s failed! \n%s" % (pgnfile, e))
def do_import(self, filename):
print(filename)
# collect new names not in they dict yet
self.collection_data = []
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
zf = zipfile.ZipFile(filename, "r")
files = [f for f in zf.namelist() if f.lower().endswith(".pgn")]
else:
zf = None
files = [filename]
for pgnfile in files:
if zf is None:
cf = pgn_load(open(pgnfile, "rU"))
else:
cf = pgn_load(zf.open(pgnfile, "rU"))
# use transaction to avoid autocommit slowness
trans = self.conn.begin()
try:
for i, game in enumerate(cf.games):
#print i+1#, cf.get_player_names(i)
movelist = array("H")
comments = []
cf.error = None
fenstr = cf._getTag(i, "FEN")
variant = cf.get_variant(i)
# Fixes for some non statndard Chess960 .pgn
if variant==0 and (fenstr is not None) and "Chess960" in cf._getTag(i,"Event"):
cf.tagcache[i]["Variant"] = "Fischerandom"
variant = 1
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
board = LBoard(FISCHERRANDOMCHESS)
else:
board = LBoard()
if fenstr:
try:
board.applyFen(fenstr)
except SyntaxError as e:
print(_("The game #%s can't be loaded, because of an error parsing FEN") % (i+1), e.args[0])
continue
else:
board = LBoard_FEN_START.clone()
boards = [board]
movetext = cf.get_movetext(i)
boards = cf.parse_string(movetext, boards[0], -1)
if cf.error is not None:
print("ERROR in game #%s" % (i+1), cf.error.args[0])
continue
walk(boards[0], movelist, comments)
if not movelist:
if (not comments) and (cf._getTag(i, 'White') is None) and (cf._getTag(i, 'Black') is None):
print("empty game")
continue
event_id = self.get_id(cf._getTag(i, 'Event'), event, EVENT)
site_id = self.get_id(cf._getTag(i, 'Site'), site, SITE)
game_date = cf._getTag(i, 'Date')
if game_date and not '?' in game_date:
ymd = game_date.split('.')
if len(ymd) == 3:
game_year, game_month, game_day = map(int, ymd)
else:
game_year, game_month, game_day = int(game_date[:4]), None, None
elif game_date and not '?' in game_date[:4]:
game_year, game_month, game_day = int(game_date[:4]), None, None
else:
game_year, game_month, game_day = None, None, None
game_round = cf._getTag(i, 'Round')
white, black = cf.get_player_names(i)
white_id = self.get_id(white, player, PLAYER)
black_id = self.get_id(black, player, PLAYER)
result = cf.get_result(i)
white_elo = cf._getTag(i, 'WhiteElo')
white_elo = int(white_elo) if white_elo and white_elo.isdigit() else None
black_elo = cf._getTag(i, 'BlackElo')
black_elo = int(black_elo) if black_elo and black_elo.isdigit() else None
ply_count = cf._getTag(i, "PlyCount")
event_date = cf._getTag(i, 'EventDate')
eco = cf._getTag(i, "ECO")
eco = eco[:3] if eco else None
fen = cf._getTag(i, "FEN")
variant = cf.get_variant(i)
board = cf._getTag(i, "Board")
annotator = cf._getTag(i, "Annotator")
annotator_id = self.get_id(annotator, annotator, ANNOTATOR)
collection_id = self.get_id(unicode(pgnfile), collection, COLLECTION)
self.game_data.append({
'event_id': event_id,
'site_id': site_id,
'date_year': game_year,
'date_month': game_month,
'date_day': game_day,
'round': game_round,
'white_id': white_id,
'black_id': black_id,
'result': result,
'white_elo': white_elo,
'black_elo': black_elo,
'ply_count': ply_count,
'eco': eco,
'fen': fen,
'variant': variant,
'board': board,
'annotator_id': annotator_id,
'collection_id': collection_id,
'movelist': movelist.tostring(),
'comments': unicode("|".join(comments)),
})
if len(self.game_data) >= CHUNK:
if self.collection_data:
self.conn.execute(self.ins_collection, self.collection_data)
self.collection_data = []
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
print(pgnfile, i+1)
if self.collection_data:
self.conn.execute(self.ins_collection, self.collection_data)
self.collection_data = []
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
print(pgnfile, i+1)
trans.commit()
except ProgrammingError as e:
trans.rollback()
print("Importing %s failed! %s" % (file, e))
from pychess.compat import unicode
from pychess.Utils.const import *
from pychess.Utils.lutils.LBoard import LBoard
from pychess.Savers.ChessFile import LoadingError
from pychess.Savers.pgnbase import pgn_load
from pychess.Database.dbwalk import walk
from pychess.Database.model import engine, metadata, collection, event,\
site, player, game, annotator, ini_collection
CHUNK = 1000
EVENT, SITE, PLAYER, ANNOTATOR, COLLECTION = range(5)
removeDic = {
ord(unicode("'")): None,
ord(unicode(",")): None,
ord(unicode(".")): None,
ord(unicode("-")): None,
ord(unicode(" ")): None,
}
LBoard_FEN_START = LBoard()
LBoard_FEN_START.applyFen(FEN_START)
class PgnImport():
def __init__(self):
self.conn = engine.connect()
self.ins_collection = collection.insert()
self.ins_event = event.insert()
def ellipsize (string, maxlen):
if len(string) <= maxlen or maxlen < 4:
return string
return string[:maxlen-1] + unicode("…")
from pychess.compat import unicode
from pychess.Utils.const import *
from pychess.Utils.lutils.LBoard import LBoard
from pychess.Savers.ChessFile import LoadingError
from pychess.Savers.pgnbase import pgn_load
from pychess.Database.dbwalk import walk
from pychess.Database.model import engine, metadata, collection, event,\
site, player, game, annotator, ini_collection
CHUNK = 1000
EVENT, SITE, PLAYER, ANNOTATOR, COLLECTION = range(5)
removeDic = {
ord(unicode("'")): None,
ord(unicode(",")): None,
ord(unicode(".")): None,
ord(unicode("-")): None,
ord(unicode(" ")): None,
}
LBoard_FEN_START = LBoard()
LBoard_FEN_START.applyFen(FEN_START)
class PgnImport():
def __init__(self):
self.conn = engine.connect()
self.ins_collection = collection.insert()
def do_import(self, filename):
print(filename)
# collect new names not in they dict yet
self.collection_data = []
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
zf = zipfile.ZipFile(filename, "r")
files = [f for f in zf.namelist() if f.lower().endswith(".pgn")]
else:
zf = None
files = [filename]
for pgnfile in files:
if zf is None:
cf = pgn_load(open(pgnfile, "rU"))
else:
cf = pgn_load(zf.open(pgnfile, "rU"))
# use transaction to avoid autocommit slowness
trans = self.conn.begin()
try:
for i, game in enumerate(cf.games):
#print i+1#, cf.get_player_names(i)
movelist = array("H")
comments = []
cf.error = None
fenstr = cf._getTag(i, "FEN")
variant = cf.get_variant(i)
# Fixes for some non statndard Chess960 .pgn
if variant == 0 and (
fenstr is not None) and "Chess960" in cf._getTag(
i, "Event"):
cf.tagcache[i]["Variant"] = "Fischerandom"
variant = 1
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
board = LBoard(FISCHERRANDOMCHESS)
else:
board = LBoard()
if fenstr:
try:
board.applyFen(fenstr)
except SyntaxError as e:
print(
_("The game #%s can't be loaded, because of an error parsing FEN"
) % (i + 1), e.args[0])
continue
else:
board = LBoard_FEN_START.clone()
boards = [board]
movetext = cf.get_movetext(i)
boards = cf.parse_string(movetext, boards[0], -1)
if cf.error is not None:
print("ERROR in game #%s" % (i + 1), cf.error.args[0])
continue
walk(boards[0], movelist, comments)
if not movelist:
if (not comments) and (cf._getTag(i, 'White') is
None) and (cf._getTag(
i, 'Black') is None):
print("empty game")
continue
event_id = self.get_id(cf._getTag(i, 'Event'), event,
EVENT)
site_id = self.get_id(cf._getTag(i, 'Site'), site, SITE)
game_date = cf._getTag(i, 'Date')
if game_date and not '?' in game_date:
ymd = game_date.split('.')
if len(ymd) == 3:
game_year, game_month, game_day = map(int, ymd)
else:
game_year, game_month, game_day = int(
game_date[:4]), None, None
elif game_date and not '?' in game_date[:4]:
game_year, game_month, game_day = int(
game_date[:4]), None, None
else:
game_year, game_month, game_day = None, None, None
game_round = cf._getTag(i, 'Round')
white, black = cf.get_player_names(i)
white_id = self.get_id(white, player, PLAYER)
black_id = self.get_id(black, player, PLAYER)
result = cf.get_result(i)
white_elo = cf._getTag(i, 'WhiteElo')
white_elo = int(
white_elo
) if white_elo and white_elo.isdigit() else None
black_elo = cf._getTag(i, 'BlackElo')
black_elo = int(
black_elo
) if black_elo and black_elo.isdigit() else None
ply_count = cf._getTag(i, "PlyCount")
event_date = cf._getTag(i, 'EventDate')
eco = cf._getTag(i, "ECO")
eco = eco[:3] if eco else None
fen = cf._getTag(i, "FEN")
variant = cf.get_variant(i)
board = cf._getTag(i, "Board")
annotator = cf._getTag(i, "Annotator")
annotator_id = self.get_id(annotator, annotator, ANNOTATOR)
collection_id = self.get_id(unicode(pgnfile), collection,
COLLECTION)
self.game_data.append({
'event_id':
event_id,
'site_id':
site_id,
'date_year':
game_year,
'date_month':
game_month,
'date_day':
game_day,
'round':
game_round,
'white_id':
white_id,
'black_id':
black_id,
'result':
result,
'white_elo':
white_elo,
'black_elo':
black_elo,
'ply_count':
ply_count,
'eco':
eco,
'fen':
fen,
'variant':
variant,
'board':
board,
'annotator_id':
annotator_id,
'collection_id':
collection_id,
'movelist':
movelist.tostring(),
'comments':
unicode("|".join(comments)),
})
if len(self.game_data) >= CHUNK:
if self.collection_data:
self.conn.execute(self.ins_collection,
self.collection_data)
self.collection_data = []
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player,
self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator,
self.annotator_data)
self.annotator_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
print(pgnfile, i + 1)
if self.collection_data:
self.conn.execute(self.ins_collection,
self.collection_data)
self.collection_data = []
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
print(pgnfile, i + 1)
trans.commit()
except ProgrammingError as e:
trans.rollback()
print("Importing %s failed! %s" % (file, e))
def do_import(self, filename, info=None, progressbar=None):
self.progressbar = progressbar
orig_filename = filename
count_source = self.conn.execute(
self.count_source.where(source.c.name == orig_filename)).scalar()
if count_source > 0:
print("%s is already imported" % filename)
return
# collect new names not in they dict yet
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
self.source_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
self.tag_game_data = []
if filename.startswith("http"):
filename = download_file(filename, progressbar=progressbar)
if filename is None:
return
else:
if not os.path.isfile(filename):
print("Can't open %s" % filename)
return
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
with zipfile.ZipFile(filename, "r") as zf:
path = os.path.dirname(filename)
files = [
os.path.join(path, f) for f in zf.namelist()
if f.lower().endswith(".pgn")
]
zf.extractall(path)
else:
files = [filename]
for pgnfile in files:
base_offset = self.chessfile.size if self.append_pgn else 0
basename = os.path.basename(pgnfile)
if progressbar is not None:
GLib.idle_add(progressbar.set_text,
"Reading %s ..." % basename)
else:
print("Reading %s ..." % pgnfile)
size = os.path.getsize(pgnfile)
handle = protoopen(pgnfile)
# estimated game count
all_games = max(size / 840, 1)
get_id = self.get_id
# use transaction to avoid autocommit slowness
trans = self.conn.begin()
try:
i = 0
for offs, tags in read_games(handle):
if not tags:
print("Empty game #%s" % (i + 1))
continue
if self.cancel:
trans.rollback()
return
fenstr = tags.get("FEN")
variant = tags.get("Variant")
if variant:
if "fischer" in variant.lower() or "960" in variant:
variant = "Fischerandom"
else:
variant = variant.lower().capitalize()
# Fixes for some non statndard Chess960 .pgn
if fenstr and variant == "Fischerandom":
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
if variant not in name2variant:
print("Unknown variant: %s" % variant)
continue
variant = name2variant[variant].variant
if variant == NORMALCHESS:
# lichess uses tag [Variant "Standard"]
variant = 0
else:
variant = 0
white = tags.get('White')
black = tags.get('Black')
event_id = get_id(tags.get('Event'), event, EVENT)
site_id = get_id(tags.get('Site'), site, SITE)
game_date = tags['Date'].strip()
try:
if game_date and '?' not in game_date:
ymd = game_date.split('.')
if len(ymd) == 3:
game_year, game_month, game_day = map(int, ymd)
else:
game_year, game_month, game_day = int(
game_date[:4]), None, None
elif game_date and '?' not in game_date[:4]:
game_year, game_month, game_day = int(
game_date[:4]), None, None
else:
game_year, game_month, game_day = None, None, None
except:
game_year, game_month, game_day = None, None, None
game_round = tags.get('Round')
white_id = get_id(unicode(white), player, PLAYER)
black_id = get_id(unicode(black), player, PLAYER)
result = tags.get("Result")
if result in pgn2Const:
result = pgn2Const[result]
else:
print("Invalid Result tag in game #%s: %s" %
(i + 1, result))
continue
white_elo = tags.get('WhiteElo')
white_elo = int(
white_elo
) if white_elo and white_elo.isdigit() else None
black_elo = tags.get('BlackElo')
black_elo = int(
black_elo
) if black_elo and black_elo.isdigit() else None
time_control = tags.get("TimeControl")
eco = tags.get("ECO")
eco = eco[:3] if eco else None
fen = tags.get("FEN")
board_tag = tags.get("Board")
annotator_id = get_id(tags.get("Annotator"), annotator,
ANNOTATOR)
source_id = get_id(unicode(orig_filename),
source,
SOURCE,
info=info)
self.next_id[GAME] += 1
ply_count = tags.get("PlyCount")
self.game_data.append({
'offset': base_offset + int(offs),
'offset8': (int(offs) >> 3) << 3,
'event_id': event_id,
'site_id': site_id,
'date_year': game_year,
'date_month': game_month,
'date_day': game_day,
'round': game_round,
'white_id': white_id,
'black_id': black_id,
'result': result,
'white_elo': white_elo,
'black_elo': black_elo,
'ply_count': ply_count,
'eco': eco,
'fen': fen,
'variant': variant,
'board': board_tag,
'time_control': time_control,
'annotator_id': annotator_id,
'source_id': source_id,
})
i += 1
if len(self.game_data) >= self.CHUNK:
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player,
self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator,
self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source,
self.source_data)
self.source_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
i / float(all_games))
GLib.idle_add(
progressbar.set_text,
"%s game headers from %s imported" %
(i, basename))
else:
print(pgnfile, i)
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source, self.source_data)
self.source_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
i / float(all_games))
GLib.idle_add(
progressbar.set_text,
"%s game headers from %s imported" % (i, basename))
else:
print(pgnfile, i)
trans.commit()
if self.append_pgn:
# reopen database to write
self.db_handle.close()
self.db_handle = protosave(self.chessfile.path,
self.append_pgn)
print("Append from %s to %s" %
(pgnfile, self.chessfile.path))
handle.seek(0)
for line in handle:
self.db_handle.write(line)
self.db_handle.close()
self.chessfile.handle = protoopen(self.chessfile.path)
except SQLAlchemyError as e:
trans.rollback()
print("Importing %s failed! \n%s" % (pgnfile, e))
def do_import(self, filename, info=None, progressbar=None):
self.progressbar = progressbar
if progressbar is not None:
self.pulse = True
self.timeout_id = GObject.timeout_add(50, self.on_timeout, None)
# collect new names not in they dict yet
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
self.source_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
self.bitboard_data = []
self.tag_game_data = []
if filename.startswith("http"):
filename = download_file(filename, progressbar=progressbar)
if filename is None:
return
else:
if not os.path.isfile(filename):
print("Can't open %s" % filename)
return
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
zf = zipfile.ZipFile(filename, "r")
files = [f for f in zf.namelist() if f.lower().endswith(".pgn")]
else:
zf = None
files = [filename]
for pgnfile in files:
basename = os.path.basename(pgnfile)
if progressbar is not None:
GLib.idle_add(progressbar.set_text,
"Reading %s ..." % basename)
else:
print("Reading %s ..." % pgnfile)
if zf is None:
cf = pgn_load(protoopen(pgnfile))
else:
pgn_file = io.TextIOWrapper(zf.open(pgnfile),
encoding=PGN_ENCODING,
newline='')
cf = pgn_load(pgn_file)
if progressbar is not None:
self.pulse = False
all_games = len(cf.games)
self.CHUNK = 1000 if all_games > 5000 else 100
get_tag = cf._getTag
get_id = self.get_id
# use transaction to avoid autocommit slowness
trans = self.conn.begin()
try:
for i in range(all_games):
# print i+1#, cf.get_player_names(i)
if self.cancel:
trans.rollback()
return
fenstr = get_tag(i, "FEN")
variant = cf.get_variant(i)
# Fixes for some non statndard Chess960 .pgn
if fenstr and variant == "Fischerandom":
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
if variant not in name2variant:
print("Unknown variant: %s" % variant)
continue
variant = name2variant[variant].variant
board = LBoard(variant)
else:
variant = 0
board = START_BOARD.clone()
if fenstr:
try:
board.applyFen(fenstr)
except SyntaxError as e:
print(
_("The game #%s can't be loaded, because of an error parsing FEN"
) % (i + 1), e.args[0])
continue
elif variant:
board.applyFen(FEN_START)
movetext = cf.get_movetext(i)
movelist = array("H")
comments = []
cf.error = None
# First we try to use simple_parse_movetext()
# assuming most games in .pgn contains only moves
# without any comments/variations
simple = False
if not fenstr and not variant:
bitboards = []
simple = cf.simple_parse_movetext(
movetext, board, movelist, bitboards)
if cf.error is not None:
print("ERROR in game #%s" % (i + 1),
cf.error.args[0])
continue
# If simple_parse_movetext() find any comments/variations
# we restart parsing with full featured parse_movetext()
if not simple:
movelist = array("H")
bitboards = None
# in case simple_parse_movetext failed we have to reset our lboard
if not fenstr and not variant:
board = START_BOARD.clone()
# parse movetext to create boards tree structure
boards = [board]
boards = cf.parse_movetext(movetext,
boards[0],
-1,
pgn_import=True)
if cf.error is not None:
print("ERROR in game #%s" % (i + 1),
cf.error.args[0])
continue
# create movelist and comments from boards tree
walk(boards[0], movelist, comments)
white = get_tag(i, 'White')
black = get_tag(i, 'Black')
if not movelist:
if (not comments) and (not white) and (not black):
print("empty game")
continue
event_id = get_id(get_tag(i, 'Event'), event, EVENT)
site_id = get_id(get_tag(i, 'Site'), site, SITE)
game_date = get_tag(i, 'Date').strip()
try:
if game_date and '?' not in game_date:
ymd = game_date.split('.')
if len(ymd) == 3:
game_year, game_month, game_day = map(int, ymd)
else:
game_year, game_month, game_day = int(
game_date[:4]), None, None
elif game_date and '?' not in game_date[:4]:
game_year, game_month, game_day = int(
game_date[:4]), None, None
except:
game_year, game_month, game_day = None, None, None
game_round = get_tag(i, 'Round')
white_fide_id = get_tag(i, 'WhiteFideId')
black_fide_id = get_tag(i, 'BlackFideId')
white_id = get_id(unicode(white),
player,
PLAYER,
fide_id=white_fide_id)
black_id = get_id(unicode(black),
player,
PLAYER,
fide_id=black_fide_id)
result = cf.get_result(i)
white_elo = get_tag(i, 'WhiteElo')
white_elo = int(
white_elo
) if white_elo and white_elo.isdigit() else None
black_elo = get_tag(i, 'BlackElo')
black_elo = int(
black_elo
) if black_elo and black_elo.isdigit() else None
ply_count = get_tag(i, "PlyCount")
time_control = get_tag(i, "TimeControl")
eco = get_tag(i, "ECO")
eco = eco[:3] if eco else None
fen = get_tag(i, "FEN")
board_tag = get_tag(i, "Board")
annotator_id = get_id(get_tag(i, "Annotator"), annotator,
ANNOTATOR)
source_id = get_id(unicode(pgnfile),
source,
SOURCE,
info=info)
game_id = self.next_id[GAME]
self.next_id[GAME] += 1
# annotated game
if bitboards is None:
for ply, board in enumerate(boards):
bb = board.friends[0] | board.friends[1]
# Avoid to include mate in x .pgn collections and similar in opening tree
if fen and "/pppppppp/8/8/8/8/PPPPPPPP/" not in fen:
ply = -1
self.bitboard_data.append({
'game_id':
game_id,
'ply':
ply,
'bitboard':
bb - DB_MAXINT_SHIFT,
})
# simple game
else:
for ply, bb in enumerate(bitboards):
self.bitboard_data.append({
'game_id':
game_id,
'ply':
ply,
'bitboard':
bb - DB_MAXINT_SHIFT,
})
self.game_data.append({
'event_id':
event_id,
'site_id':
site_id,
'date_year':
game_year,
'date_month':
game_month,
'date_day':
game_day,
'round':
game_round,
'white_id':
white_id,
'black_id':
black_id,
'result':
result,
'white_elo':
white_elo,
'black_elo':
black_elo,
'ply_count':
ply_count,
'eco':
eco,
'fen':
fen,
'variant':
variant,
'board':
board_tag,
'time_control':
time_control,
'annotator_id':
annotator_id,
'source_id':
source_id,
'movelist':
movelist.tostring(),
'comments':
unicode("|".join(comments)),
})
if len(self.game_data) >= self.CHUNK:
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player,
self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator,
self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source,
self.source_data)
self.source_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if self.bitboard_data:
self.conn.execute(self.ins_bitboard,
self.bitboard_data)
self.bitboard_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
(i + 1) / float(all_games))
GLib.idle_add(
progressbar.set_text,
"%s / %s from %s imported" %
(i + 1, all_games, basename))
else:
print(pgnfile, i + 1)
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source, self.source_data)
self.source_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if self.bitboard_data:
self.conn.execute(self.ins_bitboard, self.bitboard_data)
self.bitboard_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
(i + 1) / float(all_games))
GLib.idle_add(
progressbar.set_text, "%s / %s from %s imported" %
(i + 1, all_games, basename))
else:
print(pgnfile, i + 1)
trans.commit()
except SQLAlchemyError as e:
trans.rollback()
print("Importing %s failed! \n%s" % (file, e))
