def do_import(self, filename):
print(filename)
# collect new names not in they dict yet
self.collection_data = []
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
zf = zipfile.ZipFile(filename, "r")
files = [f for f in zf.namelist() if f.lower().endswith(".pgn")]
else:
zf = None
files = [filename]
for pgnfile in files:
if zf is None:
cf = pgn_load(open(pgnfile, "rU"))
else:
cf = pgn_load(zf.open(pgnfile, "rU"))
# use transaction to avoid autocommit slowness
trans = self.conn.begin()
try:
for i, game in enumerate(cf.games):
#print i+1#, cf.get_player_names(i)
movelist = array("H")
comments = []
cf.error = None
fenstr = cf._getTag(i, "FEN")
variant = cf.get_variant(i)
# Fixes for some non statndard Chess960 .pgn
if variant==0 and (fenstr is not None) and "Chess960" in cf._getTag(i,"Event"):
cf.tagcache[i]["Variant"] = "Fischerandom"
variant = 1
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
board = LBoard(FISCHERRANDOMCHESS)
else:
board = LBoard()
if fenstr:
try:
board.applyFen(fenstr)
except SyntaxError as e:
print(_("The game #%s can't be loaded, because of an error parsing FEN") % (i+1), e.args[0])
continue
else:
board = LBoard_FEN_START.clone()
boards = [board]
movetext = cf.get_movetext(i)
boards = cf.parse_string(movetext, boards[0], -1)
if cf.error is not None:
print("ERROR in game #%s" % (i+1), cf.error.args[0])
continue
walk(boards[0], movelist, comments)
if not movelist:
if (not comments) and (cf._getTag(i, 'White') is None) and (cf._getTag(i, 'Black') is None):
print("empty game")
continue
event_id = self.get_id(cf._getTag(i, 'Event'), event, EVENT)
site_id = self.get_id(cf._getTag(i, 'Site'), site, SITE)
game_date = cf._getTag(i, 'Date')
if game_date and not '?' in game_date:
ymd = game_date.split('.')
if len(ymd) == 3:
game_year, game_month, game_day = map(int, ymd)
else:
game_year, game_month, game_day = int(game_date[:4]), None, None
elif game_date and not '?' in game_date[:4]:
game_year, game_month, game_day = int(game_date[:4]), None, None
else:
game_year, game_month, game_day = None, None, None
game_round = cf._getTag(i, 'Round')
white, black = cf.get_player_names(i)
white_id = self.get_id(white, player, PLAYER)
black_id = self.get_id(black, player, PLAYER)
result = cf.get_result(i)
white_elo = cf._getTag(i, 'WhiteElo')
white_elo = int(white_elo) if white_elo and white_elo.isdigit() else None
black_elo = cf._getTag(i, 'BlackElo')
black_elo = int(black_elo) if black_elo and black_elo.isdigit() else None
ply_count = cf._getTag(i, "PlyCount")
event_date = cf._getTag(i, 'EventDate')
eco = cf._getTag(i, "ECO")
eco = eco[:3] if eco else None
fen = cf._getTag(i, "FEN")
variant = cf.get_variant(i)
board = cf._getTag(i, "Board")
annotator = cf._getTag(i, "Annotator")
annotator_id = self.get_id(annotator, annotator, ANNOTATOR)
collection_id = self.get_id(unicode(pgnfile), collection, COLLECTION)
self.game_data.append({
'event_id': event_id,
'site_id': site_id,
'date_year': game_year,
'date_month': game_month,
'date_day': game_day,
'round': game_round,
'white_id': white_id,
'black_id': black_id,
'result': result,
'white_elo': white_elo,
'black_elo': black_elo,
'ply_count': ply_count,
'eco': eco,
'fen': fen,
'variant': variant,
'board': board,
'annotator_id': annotator_id,
'collection_id': collection_id,
'movelist': movelist.tostring(),
'comments': unicode("|".join(comments)),
})
if len(self.game_data) >= CHUNK:
if self.collection_data:
self.conn.execute(self.ins_collection, self.collection_data)
self.collection_data = []
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
print(pgnfile, i+1)
if self.collection_data:
self.conn.execute(self.ins_collection, self.collection_data)
self.collection_data = []
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
print(pgnfile, i+1)
trans.commit()
except ProgrammingError as e:
trans.rollback()
print("Importing %s failed! %s" % (file, e))
def save (file, model, position=None):
movelist = array("H")
comments = []
walk(model.boards[0].board, movelist, comments)
game_event = model.tags["Event"]
game_site = model.tags["Site"]
year, month, day = int(model.tags["Year"]), int(model.tags["Month"]), int(model.tags["Day"])
game_round = model.tags.get("Round")
white = repr(model.players[WHITE])
black = repr(model.players[BLACK])
result = model.status
eco = model.tags.get("ECO")
board = int(model.tags.get("Board")) if model.tags.get("Board") else None
white_elo = int(model.tags.get("WhiteElo")) if model.tags.get("WhiteElo") else None
black_elo = int(model.tags.get("BlackElo")) if model.tags.get("BlackElo") else None
variant = 1 if issubclass(model.variant, FischerandomBoard) else None
fen = model.boards[0].board.asFen()
fen = fen if fen != FEN_START else None
game_annotator = model.tags.get("Annotator")
ply_count = model.ply-model.lowply
def get_id(table, name):
if not name:
return None
s = select([table.c.id], table.c.name==name)
result = conn.execute(s)
id_ = result.scalar()
if id_ is None:
result = conn.execute(table.insert().values(name=name))
id_ = result.inserted_primary_key[0]
return id_
conn = dbmodel.engine.connect()
trans = conn.begin()
try:
event_id = get_id(event, game_event)
site_id = get_id(site, game_site)
white_id = get_id(player, white)
black_id = get_id(player, black)
annotator_id = get_id(annotator, game_annotator)
wt = model.players[WHITE].__type__
bt = model.players[BLACK].__type__
if REMOTE in (wt, bt):
collection_id = REMOTE
elif ARTIFICIAL in (wt, bt):
collection_id = ARTIFICIAL
else:
collection_id = LOCAL
new_values = {
'event_id': event_id,
'site_id': site_id,
'date_year': year,
'date_month': month,
'date_day': day,
'round': game_round,
'white_id': white_id,
'black_id': black_id,
'result': result,
'white_elo': white_elo,
'black_elo': black_elo,
'ply_count': ply_count,
'eco': eco,
'board': board,
'fen': fen,
'variant': variant,
'annotator_id': annotator_id,
'collection_id': collection_id,
'movelist': movelist.tostring(),
'comments': "|".join(comments),
}
if hasattr(model, "game_id") and model.game_id is not None:
result = conn.execute(game.update().where(game.c.id==model.game_id).values(new_values))
else:
result = conn.execute(game.insert().values(new_values))
model.game_id = result.inserted_primary_key[0]
trans.commit()
except:
trans.rollback()
raise
def do_import(self, filename):
print(filename)
# collect new names not in they dict yet
self.collection_data = []
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
zf = zipfile.ZipFile(filename, "r")
files = [f for f in zf.namelist() if f.lower().endswith(".pgn")]
else:
zf = None
files = [filename]
for pgnfile in files:
if zf is None:
cf = pgn_load(open(pgnfile, "rU"))
else:
cf = pgn_load(zf.open(pgnfile, "rU"))
# use transaction to avoid autocommit slowness
trans = self.conn.begin()
try:
for i, game in enumerate(cf.games):
#print i+1#, cf.get_player_names(i)
movelist = array("H")
comments = []
cf.error = None
fenstr = cf._getTag(i, "FEN")
variant = cf.get_variant(i)
# Fixes for some non statndard Chess960 .pgn
if variant == 0 and (
fenstr is not None) and "Chess960" in cf._getTag(
i, "Event"):
cf.tagcache[i]["Variant"] = "Fischerandom"
variant = 1
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
board = LBoard(FISCHERRANDOMCHESS)
else:
board = LBoard()
if fenstr:
try:
board.applyFen(fenstr)
except SyntaxError as e:
print(
_("The game #%s can't be loaded, because of an error parsing FEN"
) % (i + 1), e.args[0])
continue
else:
board = LBoard_FEN_START.clone()
boards = [board]
movetext = cf.get_movetext(i)
boards = cf.parse_string(movetext, boards[0], -1)
if cf.error is not None:
print("ERROR in game #%s" % (i + 1), cf.error.args[0])
continue
walk(boards[0], movelist, comments)
if not movelist:
if (not comments) and (cf._getTag(i, 'White') is
None) and (cf._getTag(
i, 'Black') is None):
print("empty game")
continue
event_id = self.get_id(cf._getTag(i, 'Event'), event,
EVENT)
site_id = self.get_id(cf._getTag(i, 'Site'), site, SITE)
game_date = cf._getTag(i, 'Date')
if game_date and not '?' in game_date:
ymd = game_date.split('.')
if len(ymd) == 3:
game_year, game_month, game_day = map(int, ymd)
else:
game_year, game_month, game_day = int(
game_date[:4]), None, None
elif game_date and not '?' in game_date[:4]:
game_year, game_month, game_day = int(
game_date[:4]), None, None
else:
game_year, game_month, game_day = None, None, None
game_round = cf._getTag(i, 'Round')
white, black = cf.get_player_names(i)
white_id = self.get_id(white, player, PLAYER)
black_id = self.get_id(black, player, PLAYER)
result = cf.get_result(i)
white_elo = cf._getTag(i, 'WhiteElo')
white_elo = int(
white_elo
) if white_elo and white_elo.isdigit() else None
black_elo = cf._getTag(i, 'BlackElo')
black_elo = int(
black_elo
) if black_elo and black_elo.isdigit() else None
ply_count = cf._getTag(i, "PlyCount")
event_date = cf._getTag(i, 'EventDate')
eco = cf._getTag(i, "ECO")
eco = eco[:3] if eco else None
fen = cf._getTag(i, "FEN")
variant = cf.get_variant(i)
board = cf._getTag(i, "Board")
annotator = cf._getTag(i, "Annotator")
annotator_id = self.get_id(annotator, annotator, ANNOTATOR)
collection_id = self.get_id(unicode(pgnfile), collection,
COLLECTION)
self.game_data.append({
'event_id':
event_id,
'site_id':
site_id,
'date_year':
game_year,
'date_month':
game_month,
'date_day':
game_day,
'round':
game_round,
'white_id':
white_id,
'black_id':
black_id,
'result':
result,
'white_elo':
white_elo,
'black_elo':
black_elo,
'ply_count':
ply_count,
'eco':
eco,
'fen':
fen,
'variant':
variant,
'board':
board,
'annotator_id':
annotator_id,
'collection_id':
collection_id,
'movelist':
movelist.tostring(),
'comments':
unicode("|".join(comments)),
})
if len(self.game_data) >= CHUNK:
if self.collection_data:
self.conn.execute(self.ins_collection,
self.collection_data)
self.collection_data = []
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player,
self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator,
self.annotator_data)
self.annotator_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
print(pgnfile, i + 1)
if self.collection_data:
self.conn.execute(self.ins_collection,
self.collection_data)
self.collection_data = []
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
print(pgnfile, i + 1)
trans.commit()
except ProgrammingError as e:
trans.rollback()
print("Importing %s failed! %s" % (file, e))
def do_import(self, filename, info=None, progressbar=None):
DB_MAXINT_SHIFT = get_maxint_shift(self.engine)
self.progressbar = progressbar
orig_filename = filename
count_source = self.conn.execute(
self.count_source.where(source.c.name == orig_filename)).scalar()
if count_source > 0:
print("%s is already imported" % filename)
return
# collect new names not in they dict yet
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
self.source_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
self.bitboard_data = []
self.stat_ins_data = []
self.stat_upd_data = []
self.tag_game_data = []
if filename.startswith("http"):
filename = download_file(filename, progressbar=progressbar)
if filename is None:
return
else:
if not os.path.isfile(filename):
print("Can't open %s" % filename)
return
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
zf = zipfile.ZipFile(filename, "r")
files = [f for f in zf.namelist() if f.lower().endswith(".pgn")]
else:
zf = None
files = [filename]
for pgnfile in files:
basename = os.path.basename(pgnfile)
if progressbar is not None:
GLib.idle_add(progressbar.set_text,
"Reading %s ..." % basename)
else:
print("Reading %s ..." % pgnfile)
if zf is None:
size = os.path.getsize(pgnfile)
handle = protoopen(pgnfile)
else:
size = zf.getinfo(pgnfile).file_size
handle = io.TextIOWrapper(zf.open(pgnfile),
encoding=PGN_ENCODING,
newline='')
cf = PgnBase(handle, [])
# estimated game count
all_games = max(size / 840, 1)
self.CHUNK = 1000 if all_games > 5000 else 100
get_id = self.get_id
# use transaction to avoid autocommit slowness
trans = self.conn.begin()
try:
i = 0
for tagtext, movetext in read_games(handle):
tags = defaultdict(str, tagre.findall(tagtext))
if not tags:
print("Empty game #%s" % (i + 1))
continue
if self.cancel:
trans.rollback()
return
fenstr = tags.get("FEN")
variant = tags.get("Variant")
if variant:
if "fischer" in variant.lower() or "960" in variant:
variant = "Fischerandom"
else:
variant = variant.lower().capitalize()
# Fixes for some non statndard Chess960 .pgn
if fenstr and variant == "Fischerandom":
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
if variant not in name2variant:
print("Unknown variant: %s" % variant)
continue
variant = name2variant[variant].variant
if variant == NORMALCHESS:
# lichess uses tag [Variant "Standard"]
variant = 0
board = START_BOARD.clone()
else:
board = LBoard(variant)
elif fenstr:
variant = 0
board = LBoard()
else:
variant = 0
board = START_BOARD.clone()
if fenstr:
try:
board.applyFen(fenstr)
except SyntaxError as e:
print(
_("The game #%s can't be loaded, because of an error parsing FEN"
) % (i + 1), e.args[0])
continue
elif variant:
board.applyFen(FEN_START)
movelist = array("H")
comments = []
cf.error = None
# First we try to use simple_parse_movetext()
# assuming most games in .pgn contains only moves
# without any comments/variations
simple = False
if not fenstr and not variant:
bitboards = []
simple = cf.simple_parse_movetext(
movetext, board, movelist, bitboards)
if cf.error is not None:
print("ERROR in %s game #%s" % (pgnfile, i + 1),
cf.error.args[0])
continue
# If simple_parse_movetext() find any comments/variations
# we restart parsing with full featured parse_movetext()
if not simple:
movelist = array("H")
bitboards = None
# in case simple_parse_movetext failed we have to reset our lboard
if not fenstr and not variant:
board = START_BOARD.clone()
# parse movetext to create boards tree structure
boards = [board]
boards = cf.parse_movetext(movetext,
boards[0],
-1,
pgn_import=True)
if cf.error is not None:
print("ERROR in %s game #%s" % (pgnfile, i + 1),
cf.error.args[0])
continue
# create movelist and comments from boards tree
walk(boards[0], movelist, comments)
white = tags.get('White')
black = tags.get('Black')
if not movelist:
if (not comments) and (not white) and (not black):
print("Empty game #%s" % (i + 1))
continue
event_id = get_id(tags.get('Event'), event, EVENT)
site_id = get_id(tags.get('Site'), site, SITE)
game_date = tags.get('Date').strip()
try:
if game_date and '?' not in game_date:
ymd = game_date.split('.')
if len(ymd) == 3:
game_year, game_month, game_day = map(int, ymd)
else:
game_year, game_month, game_day = int(
game_date[:4]), None, None
elif game_date and '?' not in game_date[:4]:
game_year, game_month, game_day = int(
game_date[:4]), None, None
else:
game_year, game_month, game_day = None, None, None
except:
game_year, game_month, game_day = None, None, None
game_round = tags.get('Round')
white_fide_id = tags.get('WhiteFideId')
black_fide_id = tags.get('BlackFideId')
white_id = get_id(unicode(white),
player,
PLAYER,
fide_id=white_fide_id)
black_id = get_id(unicode(black),
player,
PLAYER,
fide_id=black_fide_id)
result = tags.get("Result")
if result in pgn2Const:
result = pgn2Const[result]
else:
print("Invalid Result tag in game #%s: %s" %
(i + 1, result))
continue
white_elo = tags.get('WhiteElo')
white_elo = int(
white_elo
) if white_elo and white_elo.isdigit() else None
black_elo = tags.get('BlackElo')
black_elo = int(
black_elo
) if black_elo and black_elo.isdigit() else None
time_control = tags.get("TimeControl")
eco = tags.get("ECO")
eco = eco[:3] if eco else None
fen = tags.get("FEN")
board_tag = tags.get("Board")
annotator_id = get_id(tags.get("Annotator"), annotator,
ANNOTATOR)
source_id = get_id(unicode(orig_filename),
source,
SOURCE,
info=info)
game_id = self.next_id[GAME]
self.next_id[GAME] += 1
# annotated game
if bitboards is None:
for ply, board in enumerate(boards):
if ply == 0:
continue
bb = board.friends[0] | board.friends[1]
# Avoid to include mate in x .pgn collections and similar in opening tree
if fen and "/pppppppp/8/8/8/8/PPPPPPPP/" not in fen:
ply = -1
self.bitboard_data.append({
'game_id':
game_id,
'ply':
ply,
'bitboard':
bb - DB_MAXINT_SHIFT,
})
if ply <= STAT_PLY_MAX:
self.stat_ins_data.append({
'ply': ply,
'bitboard': bb - DB_MAXINT_SHIFT,
'count': 0,
'whitewon': 0,
'blackwon': 0,
'draw': 0,
'white_elo_count': 0,
'black_elo_count': 0,
'white_elo': 0,
'black_elo': 0,
})
self.stat_upd_data.append({
'_ply':
ply,
'_bitboard':
bb - DB_MAXINT_SHIFT,
'_count':
1,
'_whitewon':
1 if result == WHITEWON else 0,
'_blackwon':
1 if result == BLACKWON else 0,
'_draw':
1 if result == DRAW else 0,
'_white_elo_count':
1 if white_elo is not None else 0,
'_black_elo_count':
1 if black_elo is not None else 0,
'_white_elo':
white_elo if white_elo is not None else 0,
'_black_elo':
black_elo if black_elo is not None else 0,
})
# simple game
else:
for ply, bb in enumerate(bitboards):
if ply == 0:
continue
self.bitboard_data.append({
'game_id':
game_id,
'ply':
ply,
'bitboard':
bb - DB_MAXINT_SHIFT,
})
if ply <= STAT_PLY_MAX:
self.stat_ins_data.append({
'ply': ply,
'bitboard': bb - DB_MAXINT_SHIFT,
'count': 0,
'whitewon': 0,
'blackwon': 0,
'draw': 0,
'white_elo_count': 0,
'black_elo_count': 0,
'white_elo': 0,
'black_elo': 0,
})
self.stat_upd_data.append({
'_ply':
ply,
'_bitboard':
bb - DB_MAXINT_SHIFT,
'_count':
1,
'_whitewon':
1 if result == WHITEWON else 0,
'_blackwon':
1 if result == BLACKWON else 0,
'_draw':
1 if result == DRAW else 0,
'_white_elo_count':
1 if white_elo is not None else 0,
'_black_elo_count':
1 if black_elo is not None else 0,
'_white_elo':
white_elo if white_elo is not None else 0,
'_black_elo':
black_elo if black_elo is not None else 0,
})
ply_count = tags.get("PlyCount")
if not ply_count and not fen:
ply_count = len(
bitboards) if bitboards is not None else len(
boards)
self.game_data.append({
'event_id':
event_id,
'site_id':
site_id,
'date_year':
game_year,
'date_month':
game_month,
'date_day':
game_day,
'round':
game_round,
'white_id':
white_id,
'black_id':
black_id,
'result':
result,
'white_elo':
white_elo,
'black_elo':
black_elo,
'ply_count':
ply_count,
'eco':
eco,
'fen':
fen,
'variant':
variant,
'board':
board_tag,
'time_control':
time_control,
'annotator_id':
annotator_id,
'source_id':
source_id,
'movelist':
movelist.tostring(),
'comments':
unicode("|".join(comments)),
})
i += 1
if len(self.game_data) >= self.CHUNK:
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player,
self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator,
self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source,
self.source_data)
self.source_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if self.bitboard_data:
self.conn.execute(self.ins_bitboard,
self.bitboard_data)
self.bitboard_data = []
self.conn.execute(self.ins_stat,
self.stat_ins_data)
self.conn.execute(self.upd_stat,
self.stat_upd_data)
self.stat_ins_data = []
self.stat_upd_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
i / float(all_games))
GLib.idle_add(
progressbar.set_text,
"%s games from %s imported" % (i, basename))
else:
print(pgnfile, i)
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source, self.source_data)
self.source_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if self.bitboard_data:
self.conn.execute(self.ins_bitboard, self.bitboard_data)
self.bitboard_data = []
self.conn.execute(self.ins_stat, self.stat_ins_data)
self.conn.execute(self.upd_stat, self.stat_upd_data)
self.stat_ins_data = []
self.stat_upd_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
i / float(all_games))
GLib.idle_add(progressbar.set_text,
"%s games from %s imported" % (i, basename))
else:
print(pgnfile, i)
trans.commit()
except SQLAlchemyError as e:
trans.rollback()
print("Importing %s failed! \n%s" % (pgnfile, e))
def save(path, model, position=None):
movelist = array("H")
comments = []
walk(model.boards[0].board, movelist, comments)
game_event = model.tags["Event"]
game_site = model.tags["Site"]
year, month, day = int(model.tags["Year"]), int(model.tags["Month"]), int(
model.tags["Day"])
game_round = model.tags.get("Round")
white = repr(model.players[WHITE])
black = repr(model.players[BLACK])
result = model.status
eco = model.tags.get("ECO")
time_control = model.tags.get("TimeControl")
board = int(model.tags.get("Board")) if model.tags.get("Board") else None
white_elo = int(
model.tags.get("WhiteElo")) if model.tags.get("WhiteElo") else None
black_elo = int(
model.tags.get("BlackElo")) if model.tags.get("BlackElo") else None
variant = model.variant.variant
fen = model.boards[0].board.asFen()
fen = fen if fen != FEN_START else None
game_annotator = model.tags.get("Annotator")
ply_count = model.ply - model.lowply
def get_id(table, name):
if not name:
return None
selection = select([table.c.id], table.c.name == unicode(name))
result = conn.execute(selection)
id_ = result.scalar()
if id_ is None:
result = conn.execute(table.insert().values(name=unicode(name)))
id_ = result.inserted_primary_key[0]
return id_
engine = dbmodel.get_engine(path)
DB_MAXINT_SHIFT = get_maxint_shift(engine)
conn = engine.connect()
trans = conn.begin()
try:
event_id = get_id(event, game_event)
site_id = get_id(site, game_site)
white_id = get_id(player, white)
black_id = get_id(player, black)
annotator_id = get_id(annotator, game_annotator)
new_values = {
'event_id': event_id,
'site_id': site_id,
'date_year': year,
'date_month': month,
'date_day': day,
'round': game_round,
'white_id': white_id,
'black_id': black_id,
'result': result,
'white_elo': white_elo,
'black_elo': black_elo,
'ply_count': ply_count,
'eco': eco,
'time_control': time_control,
'board': board,
'fen': fen,
'variant': variant,
'annotator_id': annotator_id,
'movelist': movelist.tostring(),
'comments': "|".join(comments),
}
if hasattr(model, "game_id") and model.game_id is not None:
result = conn.execute(game.update().where(
game.c.id == model.game_id).values(new_values))
# TODO: ?
result = conn.execute(
bitboard.delete().where(bitboard.c.game_id == model.game_id))
else:
result = conn.execute(game.insert().values(new_values))
game_id = model.game_id = result.inserted_primary_key[0]
if not fen:
bitboard_data = []
stat_ins_data = []
stat_upd_data = []
result = model.status
for ply, board in enumerate(model.boards):
if ply == 0:
continue
bb = board.board.friends[0] | board.board.friends[1]
bitboard_data.append({
'game_id': game_id,
'ply': ply,
'bitboard': bb - DB_MAXINT_SHIFT,
})
if ply <= STAT_PLY_MAX:
stat_ins_data.append({
'ply': ply,
'bitboard': bb - DB_MAXINT_SHIFT,
'count': 0,
'whitewon': 0,
'blackwon': 0,
'draw': 0,
'white_elo_count': 0,
'black_elo_count': 0,
'white_elo': 0,
'black_elo': 0,
})
stat_upd_data.append({
'_ply':
ply,
'_bitboard':
bb - DB_MAXINT_SHIFT,
'_count':
1,
'_whitewon':
1 if result == WHITEWON else 0,
'_blackwon':
1 if result == BLACKWON else 0,
'_draw':
1 if result == DRAW else 0,
'_white_elo_count':
1 if white_elo is not None else 0,
'_black_elo_count':
1 if black_elo is not None else 0,
'_white_elo':
white_elo if white_elo is not None else 0,
'_black_elo':
black_elo if black_elo is not None else 0,
})
result = conn.execute(bitboard.insert(), bitboard_data)
conn.execute(insert_or_ignore(engine, stat.insert()),
stat_ins_data)
conn.execute(upd_stat, stat_upd_data)
trans.commit()
except:
trans.rollback()
raise
conn.close()
def do_import(self, filename, info=None, progressbar=None):
self.progressbar = progressbar
if progressbar is not None:
self.pulse = True
self.timeout_id = GObject.timeout_add(50, self.on_timeout, None)
# collect new names not in they dict yet
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
self.source_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
self.bitboard_data = []
self.tag_game_data = []
if filename.startswith("http"):
filename = download_file(filename, progressbar=progressbar)
if filename is None:
return
else:
if not os.path.isfile(filename):
print("Can't open %s" % filename)
return
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
zf = zipfile.ZipFile(filename, "r")
files = [f for f in zf.namelist() if f.lower().endswith(".pgn")]
else:
zf = None
files = [filename]
for pgnfile in files:
basename = os.path.basename(pgnfile)
if progressbar is not None:
GLib.idle_add(progressbar.set_text,
"Reading %s ..." % basename)
else:
print("Reading %s ..." % pgnfile)
if zf is None:
cf = pgn_load(protoopen(pgnfile))
else:
pgn_file = io.TextIOWrapper(zf.open(pgnfile),
encoding=PGN_ENCODING,
newline='')
cf = pgn_load(pgn_file)
if progressbar is not None:
self.pulse = False
all_games = len(cf.games)
self.CHUNK = 1000 if all_games > 5000 else 100
get_tag = cf._getTag
get_id = self.get_id
# use transaction to avoid autocommit slowness
trans = self.conn.begin()
try:
for i in range(all_games):
# print i+1#, cf.get_player_names(i)
if self.cancel:
trans.rollback()
return
fenstr = get_tag(i, "FEN")
variant = cf.get_variant(i)
# Fixes for some non statndard Chess960 .pgn
if fenstr and variant == "Fischerandom":
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
if variant not in name2variant:
print("Unknown variant: %s" % variant)
continue
variant = name2variant[variant].variant
board = LBoard(variant)
else:
variant = 0
board = START_BOARD.clone()
if fenstr:
try:
board.applyFen(fenstr)
except SyntaxError as e:
print(
_("The game #%s can't be loaded, because of an error parsing FEN"
) % (i + 1), e.args[0])
continue
elif variant:
board.applyFen(FEN_START)
movetext = cf.get_movetext(i)
movelist = array("H")
comments = []
cf.error = None
# First we try to use simple_parse_movetext()
# assuming most games in .pgn contains only moves
# without any comments/variations
simple = False
if not fenstr and not variant:
bitboards = []
simple = cf.simple_parse_movetext(
movetext, board, movelist, bitboards)
if cf.error is not None:
print("ERROR in game #%s" % (i + 1),
cf.error.args[0])
continue
# If simple_parse_movetext() find any comments/variations
# we restart parsing with full featured parse_movetext()
if not simple:
movelist = array("H")
bitboards = None
# in case simple_parse_movetext failed we have to reset our lboard
if not fenstr and not variant:
board = START_BOARD.clone()
# parse movetext to create boards tree structure
boards = [board]
boards = cf.parse_movetext(movetext,
boards[0],
-1,
pgn_import=True)
if cf.error is not None:
print("ERROR in game #%s" % (i + 1),
cf.error.args[0])
continue
# create movelist and comments from boards tree
walk(boards[0], movelist, comments)
white = get_tag(i, 'White')
black = get_tag(i, 'Black')
if not movelist:
if (not comments) and (not white) and (not black):
print("empty game")
continue
event_id = get_id(get_tag(i, 'Event'), event, EVENT)
site_id = get_id(get_tag(i, 'Site'), site, SITE)
game_date = get_tag(i, 'Date').strip()
try:
if game_date and '?' not in game_date:
ymd = game_date.split('.')
if len(ymd) == 3:
game_year, game_month, game_day = map(int, ymd)
else:
game_year, game_month, game_day = int(
game_date[:4]), None, None
elif game_date and '?' not in game_date[:4]:
game_year, game_month, game_day = int(
game_date[:4]), None, None
except:
game_year, game_month, game_day = None, None, None
game_round = get_tag(i, 'Round')
white_fide_id = get_tag(i, 'WhiteFideId')
black_fide_id = get_tag(i, 'BlackFideId')
white_id = get_id(unicode(white),
player,
PLAYER,
fide_id=white_fide_id)
black_id = get_id(unicode(black),
player,
PLAYER,
fide_id=black_fide_id)
result = cf.get_result(i)
white_elo = get_tag(i, 'WhiteElo')
white_elo = int(
white_elo
) if white_elo and white_elo.isdigit() else None
black_elo = get_tag(i, 'BlackElo')
black_elo = int(
black_elo
) if black_elo and black_elo.isdigit() else None
ply_count = get_tag(i, "PlyCount")
time_control = get_tag(i, "TimeControl")
eco = get_tag(i, "ECO")
eco = eco[:3] if eco else None
fen = get_tag(i, "FEN")
board_tag = get_tag(i, "Board")
annotator_id = get_id(get_tag(i, "Annotator"), annotator,
ANNOTATOR)
source_id = get_id(unicode(pgnfile),
source,
SOURCE,
info=info)
game_id = self.next_id[GAME]
self.next_id[GAME] += 1
# annotated game
if bitboards is None:
for ply, board in enumerate(boards):
bb = board.friends[0] | board.friends[1]
# Avoid to include mate in x .pgn collections and similar in opening tree
if fen and "/pppppppp/8/8/8/8/PPPPPPPP/" not in fen:
ply = -1
self.bitboard_data.append({
'game_id':
game_id,
'ply':
ply,
'bitboard':
bb - DB_MAXINT_SHIFT,
})
# simple game
else:
for ply, bb in enumerate(bitboards):
self.bitboard_data.append({
'game_id':
game_id,
'ply':
ply,
'bitboard':
bb - DB_MAXINT_SHIFT,
})
self.game_data.append({
'event_id':
event_id,
'site_id':
site_id,
'date_year':
game_year,
'date_month':
game_month,
'date_day':
game_day,
'round':
game_round,
'white_id':
white_id,
'black_id':
black_id,
'result':
result,
'white_elo':
white_elo,
'black_elo':
black_elo,
'ply_count':
ply_count,
'eco':
eco,
'fen':
fen,
'variant':
variant,
'board':
board_tag,
'time_control':
time_control,
'annotator_id':
annotator_id,
'source_id':
source_id,
'movelist':
movelist.tostring(),
'comments':
unicode("|".join(comments)),
})
if len(self.game_data) >= self.CHUNK:
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player,
self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator,
self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source,
self.source_data)
self.source_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if self.bitboard_data:
self.conn.execute(self.ins_bitboard,
self.bitboard_data)
self.bitboard_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
(i + 1) / float(all_games))
GLib.idle_add(
progressbar.set_text,
"%s / %s from %s imported" %
(i + 1, all_games, basename))
else:
print(pgnfile, i + 1)
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source, self.source_data)
self.source_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if self.bitboard_data:
self.conn.execute(self.ins_bitboard, self.bitboard_data)
self.bitboard_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
(i + 1) / float(all_games))
GLib.idle_add(
progressbar.set_text, "%s / %s from %s imported" %
(i + 1, all_games, basename))
else:
print(pgnfile, i + 1)
trans.commit()
except SQLAlchemyError as e:
trans.rollback()
print("Importing %s failed! \n%s" % (file, e))