def do_import(self, filename):
print(filename)
# collect new names not in they dict yet
self.collection_data = []
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
zf = zipfile.ZipFile(filename, "r")
files = [f for f in zf.namelist() if f.lower().endswith(".pgn")]
else:
zf = None
files = [filename]
for pgnfile in files:
if zf is None:
cf = pgn_load(open(pgnfile, "rU"))
else:
cf = pgn_load(zf.open(pgnfile, "rU"))
# use transaction to avoid autocommit slowness
trans = self.conn.begin()
try:
for i, game in enumerate(cf.games):
#print i+1#, cf.get_player_names(i)
movelist = array("H")
comments = []
cf.error = None
fenstr = cf._getTag(i, "FEN")
variant = cf.get_variant(i)
# Fixes for some non statndard Chess960 .pgn
if variant == 0 and (
fenstr is not None) and "Chess960" in cf._getTag(
i, "Event"):
cf.tagcache[i]["Variant"] = "Fischerandom"
variant = 1
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
board = LBoard(FISCHERRANDOMCHESS)
else:
board = LBoard()
if fenstr:
try:
board.applyFen(fenstr)
except SyntaxError as e:
print(
_("The game #%s can't be loaded, because of an error parsing FEN"
) % (i + 1), e.args[0])
continue
else:
board = LBoard_FEN_START.clone()
boards = [board]
movetext = cf.get_movetext(i)
boards = cf.parse_string(movetext, boards[0], -1)
if cf.error is not None:
print("ERROR in game #%s" % (i + 1), cf.error.args[0])
continue
walk(boards[0], movelist, comments)
if not movelist:
if (not comments) and (cf._getTag(i, 'White') is
None) and (cf._getTag(
i, 'Black') is None):
print("empty game")
continue
event_id = self.get_id(cf._getTag(i, 'Event'), event,
EVENT)
site_id = self.get_id(cf._getTag(i, 'Site'), site, SITE)
game_date = cf._getTag(i, 'Date')
if game_date and not '?' in game_date:
ymd = game_date.split('.')
if len(ymd) == 3:
game_year, game_month, game_day = map(int, ymd)
else:
game_year, game_month, game_day = int(
game_date[:4]), None, None
elif game_date and not '?' in game_date[:4]:
game_year, game_month, game_day = int(
game_date[:4]), None, None
else:
game_year, game_month, game_day = None, None, None
game_round = cf._getTag(i, 'Round')
white, black = cf.get_player_names(i)
white_id = self.get_id(white, player, PLAYER)
black_id = self.get_id(black, player, PLAYER)
result = cf.get_result(i)
white_elo = cf._getTag(i, 'WhiteElo')
white_elo = int(
white_elo
) if white_elo and white_elo.isdigit() else None
black_elo = cf._getTag(i, 'BlackElo')
black_elo = int(
black_elo
) if black_elo and black_elo.isdigit() else None
ply_count = cf._getTag(i, "PlyCount")
event_date = cf._getTag(i, 'EventDate')
eco = cf._getTag(i, "ECO")
eco = eco[:3] if eco else None
fen = cf._getTag(i, "FEN")
variant = cf.get_variant(i)
board = cf._getTag(i, "Board")
annotator = cf._getTag(i, "Annotator")
annotator_id = self.get_id(annotator, annotator, ANNOTATOR)
collection_id = self.get_id(unicode(pgnfile), collection,
COLLECTION)
self.game_data.append({
'event_id':
event_id,
'site_id':
site_id,
'date_year':
game_year,
'date_month':
game_month,
'date_day':
game_day,
'round':
game_round,
'white_id':
white_id,
'black_id':
black_id,
'result':
result,
'white_elo':
white_elo,
'black_elo':
black_elo,
'ply_count':
ply_count,
'eco':
eco,
'fen':
fen,
'variant':
variant,
'board':
board,
'annotator_id':
annotator_id,
'collection_id':
collection_id,
'movelist':
movelist.tostring(),
'comments':
unicode("|".join(comments)),
})
if len(self.game_data) >= CHUNK:
if self.collection_data:
self.conn.execute(self.ins_collection,
self.collection_data)
self.collection_data = []
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player,
self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator,
self.annotator_data)
self.annotator_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
print(pgnfile, i + 1)
if self.collection_data:
self.conn.execute(self.ins_collection,
self.collection_data)
self.collection_data = []
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
print(pgnfile, i + 1)
trans.commit()
except ProgrammingError as e:
trans.rollback()
print("Importing %s failed! %s" % (file, e))
def do_import(self, filename):
print(filename)
# collect new names not in they dict yet
self.collection_data = []
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
zf = zipfile.ZipFile(filename, "r")
files = [f for f in zf.namelist() if f.lower().endswith(".pgn")]
else:
zf = None
files = [filename]
for pgnfile in files:
if zf is None:
cf = pgn_load(open(pgnfile, "rU"))
else:
cf = pgn_load(zf.open(pgnfile, "rU"))
# use transaction to avoid autocommit slowness
trans = self.conn.begin()
try:
for i, game in enumerate(cf.games):
#print i+1#, cf.get_player_names(i)
movelist = array("H")
comments = []
cf.error = None
fenstr = cf._getTag(i, "FEN")
variant = cf.get_variant(i)
# Fixes for some non statndard Chess960 .pgn
if variant==0 and (fenstr is not None) and "Chess960" in cf._getTag(i,"Event"):
cf.tagcache[i]["Variant"] = "Fischerandom"
variant = 1
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
board = LBoard(FISCHERRANDOMCHESS)
else:
board = LBoard()
if fenstr:
try:
board.applyFen(fenstr)
except SyntaxError as e:
print(_("The game #%s can't be loaded, because of an error parsing FEN") % (i+1), e.args[0])
continue
else:
board = LBoard_FEN_START.clone()
boards = [board]
movetext = cf.get_movetext(i)
boards = cf.parse_string(movetext, boards[0], -1)
if cf.error is not None:
print("ERROR in game #%s" % (i+1), cf.error.args[0])
continue
walk(boards[0], movelist, comments)
if not movelist:
if (not comments) and (cf._getTag(i, 'White') is None) and (cf._getTag(i, 'Black') is None):
print("empty game")
continue
event_id = self.get_id(cf._getTag(i, 'Event'), event, EVENT)
site_id = self.get_id(cf._getTag(i, 'Site'), site, SITE)
game_date = cf._getTag(i, 'Date')
if game_date and not '?' in game_date:
ymd = game_date.split('.')
if len(ymd) == 3:
game_year, game_month, game_day = map(int, ymd)
else:
game_year, game_month, game_day = int(game_date[:4]), None, None
elif game_date and not '?' in game_date[:4]:
game_year, game_month, game_day = int(game_date[:4]), None, None
else:
game_year, game_month, game_day = None, None, None
game_round = cf._getTag(i, 'Round')
white, black = cf.get_player_names(i)
white_id = self.get_id(white, player, PLAYER)
black_id = self.get_id(black, player, PLAYER)
result = cf.get_result(i)
white_elo = cf._getTag(i, 'WhiteElo')
white_elo = int(white_elo) if white_elo and white_elo.isdigit() else None
black_elo = cf._getTag(i, 'BlackElo')
black_elo = int(black_elo) if black_elo and black_elo.isdigit() else None
ply_count = cf._getTag(i, "PlyCount")
event_date = cf._getTag(i, 'EventDate')
eco = cf._getTag(i, "ECO")
eco = eco[:3] if eco else None
fen = cf._getTag(i, "FEN")
variant = cf.get_variant(i)
board = cf._getTag(i, "Board")
annotator = cf._getTag(i, "Annotator")
annotator_id = self.get_id(annotator, annotator, ANNOTATOR)
collection_id = self.get_id(unicode(pgnfile), collection, COLLECTION)
self.game_data.append({
'event_id': event_id,
'site_id': site_id,
'date_year': game_year,
'date_month': game_month,
'date_day': game_day,
'round': game_round,
'white_id': white_id,
'black_id': black_id,
'result': result,
'white_elo': white_elo,
'black_elo': black_elo,
'ply_count': ply_count,
'eco': eco,
'fen': fen,
'variant': variant,
'board': board,
'annotator_id': annotator_id,
'collection_id': collection_id,
'movelist': movelist.tostring(),
'comments': unicode("|".join(comments)),
})
if len(self.game_data) >= CHUNK:
if self.collection_data:
self.conn.execute(self.ins_collection, self.collection_data)
self.collection_data = []
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
print(pgnfile, i+1)
if self.collection_data:
self.conn.execute(self.ins_collection, self.collection_data)
self.collection_data = []
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
print(pgnfile, i+1)
trans.commit()
except ProgrammingError as e:
trans.rollback()
print("Importing %s failed! %s" % (file, e))
def do_import(self, filename, info=None, progressbar=None):
self.progressbar = progressbar
if progressbar is not None:
self.pulse = True
self.timeout_id = GObject.timeout_add(50, self.on_timeout, None)
# collect new names not in they dict yet
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
self.source_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
self.bitboard_data = []
self.tag_game_data = []
if filename.startswith("http"):
filename = download_file(filename, progressbar=progressbar)
if filename is None:
return
else:
if not os.path.isfile(filename):
print("Can't open %s" % filename)
return
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
zf = zipfile.ZipFile(filename, "r")
files = [f for f in zf.namelist() if f.lower().endswith(".pgn")]
else:
zf = None
files = [filename]
for pgnfile in files:
basename = os.path.basename(pgnfile)
if progressbar is not None:
GLib.idle_add(progressbar.set_text,
"Reading %s ..." % basename)
else:
print("Reading %s ..." % pgnfile)
if zf is None:
cf = pgn_load(protoopen(pgnfile))
else:
pgn_file = io.TextIOWrapper(zf.open(pgnfile),
encoding=PGN_ENCODING,
newline='')
cf = pgn_load(pgn_file)
if progressbar is not None:
self.pulse = False
all_games = len(cf.games)
self.CHUNK = 1000 if all_games > 5000 else 100
get_tag = cf._getTag
get_id = self.get_id
# use transaction to avoid autocommit slowness
trans = self.conn.begin()
try:
for i in range(all_games):
# print i+1#, cf.get_player_names(i)
if self.cancel:
trans.rollback()
return
fenstr = get_tag(i, "FEN")
variant = cf.get_variant(i)
# Fixes for some non statndard Chess960 .pgn
if fenstr and variant == "Fischerandom":
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
if variant not in name2variant:
print("Unknown variant: %s" % variant)
continue
variant = name2variant[variant].variant
board = LBoard(variant)
else:
variant = 0
board = START_BOARD.clone()
if fenstr:
try:
board.applyFen(fenstr)
except SyntaxError as e:
print(
_("The game #%s can't be loaded, because of an error parsing FEN"
) % (i + 1), e.args[0])
continue
elif variant:
board.applyFen(FEN_START)
movetext = cf.get_movetext(i)
movelist = array("H")
comments = []
cf.error = None
# First we try to use simple_parse_movetext()
# assuming most games in .pgn contains only moves
# without any comments/variations
simple = False
if not fenstr and not variant:
bitboards = []
simple = cf.simple_parse_movetext(
movetext, board, movelist, bitboards)
if cf.error is not None:
print("ERROR in game #%s" % (i + 1),
cf.error.args[0])
continue
# If simple_parse_movetext() find any comments/variations
# we restart parsing with full featured parse_movetext()
if not simple:
movelist = array("H")
bitboards = None
# in case simple_parse_movetext failed we have to reset our lboard
if not fenstr and not variant:
board = START_BOARD.clone()
# parse movetext to create boards tree structure
boards = [board]
boards = cf.parse_movetext(movetext,
boards[0],
-1,
pgn_import=True)
if cf.error is not None:
print("ERROR in game #%s" % (i + 1),
cf.error.args[0])
continue
# create movelist and comments from boards tree
walk(boards[0], movelist, comments)
white = get_tag(i, 'White')
black = get_tag(i, 'Black')
if not movelist:
if (not comments) and (not white) and (not black):
print("empty game")
continue
event_id = get_id(get_tag(i, 'Event'), event, EVENT)
site_id = get_id(get_tag(i, 'Site'), site, SITE)
game_date = get_tag(i, 'Date').strip()
try:
if game_date and '?' not in game_date:
ymd = game_date.split('.')
if len(ymd) == 3:
game_year, game_month, game_day = map(int, ymd)
else:
game_year, game_month, game_day = int(
game_date[:4]), None, None
elif game_date and '?' not in game_date[:4]:
game_year, game_month, game_day = int(
game_date[:4]), None, None
except:
game_year, game_month, game_day = None, None, None
game_round = get_tag(i, 'Round')
white_fide_id = get_tag(i, 'WhiteFideId')
black_fide_id = get_tag(i, 'BlackFideId')
white_id = get_id(unicode(white),
player,
PLAYER,
fide_id=white_fide_id)
black_id = get_id(unicode(black),
player,
PLAYER,
fide_id=black_fide_id)
result = cf.get_result(i)
white_elo = get_tag(i, 'WhiteElo')
white_elo = int(
white_elo
) if white_elo and white_elo.isdigit() else None
black_elo = get_tag(i, 'BlackElo')
black_elo = int(
black_elo
) if black_elo and black_elo.isdigit() else None
ply_count = get_tag(i, "PlyCount")
time_control = get_tag(i, "TimeControl")
eco = get_tag(i, "ECO")
eco = eco[:3] if eco else None
fen = get_tag(i, "FEN")
board_tag = get_tag(i, "Board")
annotator_id = get_id(get_tag(i, "Annotator"), annotator,
ANNOTATOR)
source_id = get_id(unicode(pgnfile),
source,
SOURCE,
info=info)
game_id = self.next_id[GAME]
self.next_id[GAME] += 1
# annotated game
if bitboards is None:
for ply, board in enumerate(boards):
bb = board.friends[0] | board.friends[1]
# Avoid to include mate in x .pgn collections and similar in opening tree
if fen and "/pppppppp/8/8/8/8/PPPPPPPP/" not in fen:
ply = -1
self.bitboard_data.append({
'game_id':
game_id,
'ply':
ply,
'bitboard':
bb - DB_MAXINT_SHIFT,
})
# simple game
else:
for ply, bb in enumerate(bitboards):
self.bitboard_data.append({
'game_id':
game_id,
'ply':
ply,
'bitboard':
bb - DB_MAXINT_SHIFT,
})
self.game_data.append({
'event_id':
event_id,
'site_id':
site_id,
'date_year':
game_year,
'date_month':
game_month,
'date_day':
game_day,
'round':
game_round,
'white_id':
white_id,
'black_id':
black_id,
'result':
result,
'white_elo':
white_elo,
'black_elo':
black_elo,
'ply_count':
ply_count,
'eco':
eco,
'fen':
fen,
'variant':
variant,
'board':
board_tag,
'time_control':
time_control,
'annotator_id':
annotator_id,
'source_id':
source_id,
'movelist':
movelist.tostring(),
'comments':
unicode("|".join(comments)),
})
if len(self.game_data) >= self.CHUNK:
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player,
self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator,
self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source,
self.source_data)
self.source_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if self.bitboard_data:
self.conn.execute(self.ins_bitboard,
self.bitboard_data)
self.bitboard_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
(i + 1) / float(all_games))
GLib.idle_add(
progressbar.set_text,
"%s / %s from %s imported" %
(i + 1, all_games, basename))
else:
print(pgnfile, i + 1)
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source, self.source_data)
self.source_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if self.bitboard_data:
self.conn.execute(self.ins_bitboard, self.bitboard_data)
self.bitboard_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
(i + 1) / float(all_games))
GLib.idle_add(
progressbar.set_text, "%s / %s from %s imported" %
(i + 1, all_games, basename))
else:
print(pgnfile, i + 1)
trans.commit()
except SQLAlchemyError as e:
trans.rollback()
print("Importing %s failed! \n%s" % (file, e))