def save (self, uri, saver, append, position=None):
if type(uri) == str:
fileobj = protosave(uri, append)
self.uri = uri
else:
fileobj = uri
self.uri = None
saver.save(fileobj, self, position)
self.needsSave = False
def save(self, uri, saver, append, position=None):
if isinstance(uri, basestring):
fileobj = protosave(uri, append)
self.uri = uri
else:
fileobj = uri
self.uri = None
saver.save(fileobj, self, position)
self.needsSave = False
self.emit("game_saved", uri)
def save(self, uri, saver, append, position=None):
if isinstance(uri, basestring) and not hasattr(saver, "Database"):
fileobj = protosave(uri, append)
self.uri = uri
else:
fileobj = uri
self.uri = None
saver.save(fileobj, self, position)
self.needsSave = False
self.emit("game_saved", uri)
def save(self, uri, saver, append):
if type(uri) == str:
fileobj = protosave(uri, append)
self.uri = uri
else:
fileobj = uri
self.uri = None
saver.save(fileobj, self)
self.emit("game_saved", uri)
self.needsSave = False
def save(self, uri, saver, append, position=None):
if isinstance(uri, str):
fileobj = protosave(uri, append)
self.uri = uri
else:
fileobj = uri
self.uri = None
saver.save(fileobj, self, position)
self.needsSave = False
self.emit("game_saved", uri)
def save (self, uri, saver, append):
if type(uri) == str:
fileobj = protosave(uri, append)
self.uri = uri
else:
fileobj = uri
self.uri = None
saver.save(fileobj, self)
self.emit("game_saved", uri)
self.needsSave = False
def save(self, uri, saver, append, position=None, flip=False):
if saver in (html, txt):
fileobj = open(uri,
"a" if append else "w",
encoding="utf-8",
newline="")
self.uri = uri
elif isinstance(uri, str):
fileobj = protosave(uri, append)
self.uri = uri
else:
fileobj = uri
self.uri = None
saver.save(fileobj, self, position, flip)
self.needsSave = False
self.emit("game_saved", uri)
def do_import(self, filename, info=None, progressbar=None):
self.progressbar = progressbar
orig_filename = filename
count_source = self.conn.execute(
self.count_source.where(source.c.name == orig_filename)).scalar()
if count_source > 0:
log.info("%s is already imported" % filename)
return
# collect new names not in they dict yet
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
self.source_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
self.tag_game_data = []
if filename.startswith("http"):
filename = download_file(filename, progressbar=progressbar)
if filename is None:
return
else:
if not os.path.isfile(filename):
log.info("Can't open %s" % filename)
return
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
with zipfile.ZipFile(filename, "r") as zf:
path = os.path.dirname(filename)
files = [
os.path.join(path, f) for f in zf.namelist()
if f.lower().endswith(".pgn")
]
zf.extractall(path)
else:
files = [filename]
for pgnfile in files:
base_offset = self.chessfile.size if self.append_pgn else 0
basename = os.path.basename(pgnfile)
if progressbar is not None:
GLib.idle_add(progressbar.set_text,
_("Reading %s ..." % basename))
else:
log.info("Reading %s ..." % pgnfile)
size = os.path.getsize(pgnfile)
handle = protoopen(pgnfile)
# estimated game count
all_games = max(size / 840, 1)
get_id = self.get_id
# use transaction to avoid autocommit slowness
# and to let undo importing (rollback) if self.cancel was set
trans = self.conn.begin()
try:
i = 0
for tags in read_games(handle):
if not tags:
log.info("Empty game #%s" % (i + 1))
continue
if self.cancel:
trans.rollback()
return
fenstr = tags["FEN"]
variant = tags["Variant"]
if variant:
if "fischer" in variant.lower() or "960" in variant:
variant = "Fischerandom"
else:
variant = variant.lower().capitalize()
# Fixes for some non statndard Chess960 .pgn
if fenstr and variant == "Fischerandom":
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
if variant not in name2variant:
log.info("Unknown variant: %s" % variant)
continue
variant = name2variant[variant].variant
if variant == NORMALCHESS:
# lichess uses tag [Variant "Standard"]
variant = 0
else:
variant = 0
if basename == "eco.pgn":
white = tags["Opening"]
black = tags["Variation"]
else:
white = tags["White"]
black = tags["Black"]
event_id = get_id(tags["Event"], event, EVENT)
site_id = get_id(tags["Site"], site, SITE)
date = tags["Date"]
game_round = tags['Round']
white_id = get_id(white, player, PLAYER)
black_id = get_id(black, player, PLAYER)
result = tags["Result"]
if result in pgn2Const:
result = pgn2Const[result]
else:
result = RUNNING
white_elo = tags['WhiteElo']
black_elo = tags['BlackElo']
time_control = tags["TimeControl"]
eco = tags["ECO"][:3]
fen = tags["FEN"]
board_tag = int(tags["Board"]) if "Board" in tags else 0
annotator_id = get_id(tags["Annotator"], annotator,
ANNOTATOR)
source_id = get_id(orig_filename,
source,
SOURCE,
info=info)
ply_count = tags["PlyCount"] if "PlyCount" in tags else 0
offset = base_offset + int(tags["offset"])
self.game_data.append({
'offset': offset,
'offset8': (offset >> 3) << 3,
'event_id': event_id,
'site_id': site_id,
'date': date,
'round': game_round,
'white_id': white_id,
'black_id': black_id,
'result': result,
'white_elo': white_elo,
'black_elo': black_elo,
'ply_count': ply_count,
'eco': eco,
'fen': fen,
'variant': variant,
'board': board_tag,
'time_control': time_control,
'annotator_id': annotator_id,
'source_id': source_id,
})
for tag in tags:
if tag not in dedicated_tags and tag not in other_game_tags and tags[
tag]:
self.tag_game_data.append({
'game_id':
self.next_id[GAME],
'tag_name':
tag,
'tag_value':
tags[tag],
})
self.next_id[GAME] += 1
i += 1
if len(self.game_data) >= self.CHUNK:
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player,
self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator,
self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source,
self.source_data)
self.source_data = []
if self.tag_game_data:
self.conn.execute(self.ins_tag_game,
self.tag_game_data)
self.tag_game_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
i / float(all_games))
GLib.idle_add(
progressbar.set_text,
_("%(counter)s game headers from %(filename)s imported"
% ({
"counter": i,
"filename": basename
})))
else:
log.info("From %s imported %s" % (pgnfile, i))
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source, self.source_data)
self.source_data = []
if self.tag_game_data:
self.conn.execute(self.ins_tag_game, self.tag_game_data)
self.tag_game_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
i / float(all_games))
GLib.idle_add(
progressbar.set_text,
_("%(counter)s game headers from %(filename)s imported"
% ({
"counter": i,
"filename": basename
})))
else:
log.info("From %s imported %s" % (pgnfile, i))
trans.commit()
if self.append_pgn:
# reopen database to write
self.db_handle.close()
with protosave(self.chessfile.path,
self.append_pgn) as self.db_handle:
log.info("Append from %s to %s" %
(pgnfile, self.chessfile.path))
handle.seek(0)
self.db_handle.writelines(handle)
handle.close()
if self.chessfile.scoutfish is not None:
# create new .scout from pgnfile we are importing
from pychess.Savers.pgn import scoutfish_path
args = [
scoutfish_path, "make", pgnfile,
"%s" % base_offset
]
output = subprocess.check_output(
args, stderr=subprocess.STDOUT).decode()
# append it to our existing one
if output.find("Processing...done") > 0:
old_scout = self.chessfile.scoutfish.db
new_scout = os.path.splitext(pgnfile)[0] + '.scout'
with open(old_scout,
"ab") as file1, open(new_scout,
"rb") as file2:
file1.write(file2.read())
self.chessfile.handle = protoopen(self.chessfile.path)
except SQLAlchemyError as e:
trans.rollback()
log.info("Importing %s failed! \n%s" % (pgnfile, e))
def do_import(self, filename, info=None, progressbar=None):
self.progressbar = progressbar
orig_filename = filename
count_source = self.conn.execute(
self.count_source.where(source.c.name == orig_filename)).scalar()
if count_source > 0:
print("%s is already imported" % filename)
return
# collect new names not in they dict yet
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
self.source_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
self.tag_game_data = []
if filename.startswith("http"):
filename = download_file(filename, progressbar=progressbar)
if filename is None:
return
else:
if not os.path.isfile(filename):
print("Can't open %s" % filename)
return
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
with zipfile.ZipFile(filename, "r") as zf:
path = os.path.dirname(filename)
files = [
os.path.join(path, f) for f in zf.namelist()
if f.lower().endswith(".pgn")
]
zf.extractall(path)
else:
files = [filename]
for pgnfile in files:
base_offset = self.chessfile.size if self.append_pgn else 0
basename = os.path.basename(pgnfile)
if progressbar is not None:
GLib.idle_add(progressbar.set_text,
"Reading %s ..." % basename)
else:
print("Reading %s ..." % pgnfile)
size = os.path.getsize(pgnfile)
handle = protoopen(pgnfile)
# estimated game count
all_games = max(size / 840, 1)
get_id = self.get_id
# use transaction to avoid autocommit slowness
trans = self.conn.begin()
try:
i = 0
for offs, tags in read_games(handle):
if not tags:
print("Empty game #%s" % (i + 1))
continue
if self.cancel:
trans.rollback()
return
fenstr = tags.get("FEN")
variant = tags.get("Variant")
if variant:
if "fischer" in variant.lower() or "960" in variant:
variant = "Fischerandom"
else:
variant = variant.lower().capitalize()
# Fixes for some non statndard Chess960 .pgn
if fenstr and variant == "Fischerandom":
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
if variant not in name2variant:
print("Unknown variant: %s" % variant)
continue
variant = name2variant[variant].variant
if variant == NORMALCHESS:
# lichess uses tag [Variant "Standard"]
variant = 0
else:
variant = 0
white = tags.get('White')
black = tags.get('Black')
event_id = get_id(tags.get('Event'), event, EVENT)
site_id = get_id(tags.get('Site'), site, SITE)
game_date = tags['Date'].strip()
try:
if game_date and '?' not in game_date:
ymd = game_date.split('.')
if len(ymd) == 3:
game_year, game_month, game_day = map(int, ymd)
else:
game_year, game_month, game_day = int(
game_date[:4]), None, None
elif game_date and '?' not in game_date[:4]:
game_year, game_month, game_day = int(
game_date[:4]), None, None
else:
game_year, game_month, game_day = None, None, None
except:
game_year, game_month, game_day = None, None, None
game_round = tags.get('Round')
white_id = get_id(unicode(white), player, PLAYER)
black_id = get_id(unicode(black), player, PLAYER)
result = tags.get("Result")
if result in pgn2Const:
result = pgn2Const[result]
else:
print("Invalid Result tag in game #%s: %s" %
(i + 1, result))
continue
white_elo = tags.get('WhiteElo')
white_elo = int(
white_elo
) if white_elo and white_elo.isdigit() else None
black_elo = tags.get('BlackElo')
black_elo = int(
black_elo
) if black_elo and black_elo.isdigit() else None
time_control = tags.get("TimeControl")
eco = tags.get("ECO")
eco = eco[:3] if eco else None
fen = tags.get("FEN")
board_tag = tags.get("Board")
annotator_id = get_id(tags.get("Annotator"), annotator,
ANNOTATOR)
source_id = get_id(unicode(orig_filename),
source,
SOURCE,
info=info)
self.next_id[GAME] += 1
ply_count = tags.get("PlyCount")
self.game_data.append({
'offset': base_offset + int(offs),
'offset8': (int(offs) >> 3) << 3,
'event_id': event_id,
'site_id': site_id,
'date_year': game_year,
'date_month': game_month,
'date_day': game_day,
'round': game_round,
'white_id': white_id,
'black_id': black_id,
'result': result,
'white_elo': white_elo,
'black_elo': black_elo,
'ply_count': ply_count,
'eco': eco,
'fen': fen,
'variant': variant,
'board': board_tag,
'time_control': time_control,
'annotator_id': annotator_id,
'source_id': source_id,
})
i += 1
if len(self.game_data) >= self.CHUNK:
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player,
self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator,
self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source,
self.source_data)
self.source_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
i / float(all_games))
GLib.idle_add(
progressbar.set_text,
"%s game headers from %s imported" %
(i, basename))
else:
print(pgnfile, i)
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source, self.source_data)
self.source_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
i / float(all_games))
GLib.idle_add(
progressbar.set_text,
"%s game headers from %s imported" % (i, basename))
else:
print(pgnfile, i)
trans.commit()
if self.append_pgn:
# reopen database to write
self.db_handle.close()
self.db_handle = protosave(self.chessfile.path,
self.append_pgn)
print("Append from %s to %s" %
(pgnfile, self.chessfile.path))
handle.seek(0)
for line in handle:
self.db_handle.write(line)
self.db_handle.close()
self.chessfile.handle = protoopen(self.chessfile.path)
except SQLAlchemyError as e:
trans.rollback()
print("Importing %s failed! \n%s" % (pgnfile, e))
def do_import(self, filename, info=None, progressbar=None):
self.progressbar = progressbar
orig_filename = filename
count_source = self.conn.execute(self.count_source.where(source.c.name == orig_filename)).scalar()
if count_source > 0:
print("%s is already imported" % filename)
return
# collect new names not in they dict yet
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
self.source_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
self.tag_game_data = []
if filename.startswith("http"):
filename = download_file(filename, progressbar=progressbar)
if filename is None:
return
else:
if not os.path.isfile(filename):
print("Can't open %s" % filename)
return
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
with zipfile.ZipFile(filename, "r") as zf:
path = os.path.dirname(filename)
files = [os.path.join(path, f) for f in zf.namelist() if f.lower().endswith(".pgn")]
zf.extractall(path)
else:
files = [filename]
for pgnfile in files:
base_offset = self.chessfile.size if self.append_pgn else 0
basename = os.path.basename(pgnfile)
if progressbar is not None:
GLib.idle_add(progressbar.set_text, "Reading %s ..." % basename)
else:
print("Reading %s ..." % pgnfile)
size = os.path.getsize(pgnfile)
handle = protoopen(pgnfile)
# estimated game count
all_games = max(size / 840, 1)
handle_json = None
if pgnextractor is not None:
try:
headers_json = os.path.splitext(pgnfile)[0] + ".headers.json"
if not os.path.isfile(headers_json):
output = subprocess.check_output([pgnextractor, "headers", pgnfile]).decode()
for line in output:
if line.startswith("Games"):
all_games = line.split()[1]
handle_json = protoopen(headers_json)
except subprocess.CalledProcessError:
print("pgnextractor failed")
get_id = self.get_id
# use transaction to avoid autocommit slowness
trans = self.conn.begin()
try:
i = 0
for tags in read_games(handle, handle_json):
if not tags:
print("Empty game #%s" % (i + 1))
continue
if self.cancel:
trans.rollback()
return
fenstr = tags["FEN"] if "FEN" in tags else ""
variant = tags["Variant"] if "Variant" in tags else ""
if variant:
if "fischer" in variant.lower() or "960" in variant:
variant = "Fischerandom"
else:
variant = variant.lower().capitalize()
# Fixes for some non statndard Chess960 .pgn
if fenstr and variant == "Fischerandom":
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
if variant not in name2variant:
print("Unknown variant: %s" % variant)
continue
variant = name2variant[variant].variant
if variant == NORMALCHESS:
# lichess uses tag [Variant "Standard"]
variant = 0
else:
variant = 0
white = tags["White"] if "White" in tags else ""
black = tags["Black"] if "Black" in tags else ""
event_id = get_id(tags["Event"] if "Event" in tags else "", event, EVENT)
site_id = get_id(tags["Site"] if "Site" in tags else "", site, SITE)
game_date = tags["Date"] if "Date" in tags else ""
try:
if game_date and '?' not in game_date:
ymd = game_date.split('.')
if len(ymd) == 3:
game_year, game_month, game_day = map(int, ymd)
else:
game_year, game_month, game_day = int(game_date[:4]), None, None
elif game_date and '?' not in game_date[:4]:
game_year, game_month, game_day = int(game_date[:4]), None, None
else:
game_year, game_month, game_day = None, None, None
except:
game_year, game_month, game_day = None, None, None
game_round = tags['Round'] if "Round" in tags else ""
white_id = get_id(white, player, PLAYER)
black_id = get_id(black, player, PLAYER)
result = tags["Result"] if "Result" in tags else ""
if result in pgn2Const:
result = pgn2Const[result]
else:
if basename != "eco.pgn":
print("Invalid Result tag in game #%s: %s" % (i + 1, result))
continue
white_elo = tags['WhiteElo'] if "WhiteElo" in tags else "0"
white_elo = int(white_elo) if white_elo and white_elo.isdigit() else 0
black_elo = tags['BlackElo'] if "BlackElo" in tags else ""
black_elo = int(black_elo) if black_elo and black_elo.isdigit() else 0
time_control = tags["TimeControl"] if "TimeControl" in tags else ""
eco = tags["ECO"][:3] if "ECO" in tags else ""
fen = tags["FEN"] if "FEN" in tags else ""
board_tag = int(tags["Board"]) if "Board" in tags else 0
annotator_id = get_id(tags["Annotator"] if "Annotator" in tags else "", annotator, ANNOTATOR)
source_id = get_id(orig_filename, source, SOURCE, info=info)
self.next_id[GAME] += 1
ply_count = tags["PlyCount"] if "PlyCount" in tags else 0
offset = base_offset + int(tags["offset"])
self.game_data.append({
'offset': offset,
'offset8': (offset >> 3) << 3,
'event_id': event_id,
'site_id': site_id,
'date_year': game_year,
'date_month': game_month,
'date_day': game_day,
'round': game_round,
'white_id': white_id,
'black_id': black_id,
'result': result,
'white_elo': white_elo,
'black_elo': black_elo,
'ply_count': ply_count,
'eco': eco,
'fen': fen,
'variant': variant,
'board': board_tag,
'time_control': time_control,
'annotator_id': annotator_id,
'source_id': source_id,
})
i += 1
if len(self.game_data) >= self.CHUNK:
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player,
self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator,
self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source, self.source_data)
self.source_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction, i / float(all_games))
GLib.idle_add(progressbar.set_text, "%s game headers from %s imported" % (i, basename))
else:
print(pgnfile, i)
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source, self.source_data)
self.source_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction, i / float(all_games))
GLib.idle_add(progressbar.set_text, "%s game headers from %s imported" % (i, basename))
else:
print(pgnfile, i)
trans.commit()
if self.append_pgn:
# reopen database to write
self.db_handle.close()
self.db_handle = protosave(self.chessfile.path, self.append_pgn)
print("Append from %s to %s" % (pgnfile, self.chessfile.path))
handle.seek(0)
self.db_handle.writelines(handle)
self.db_handle.close()
handle.close()
if self.chessfile.scoutfish is not None:
# create new .scout from pgnfile we are importing
from pychess.Savers.pgn import scoutfish_path
args = [scoutfish_path, "make", pgnfile, "%s" % base_offset]
output = subprocess.check_output(args, stderr=subprocess.STDOUT).decode()
# append it to our existing one
if output.find("Processing...done") > 0:
old_scout = self.chessfile.scoutfish.db
new_scout = os.path.splitext(pgnfile)[0] + '.scout'
with open(old_scout, "ab") as file1, open(new_scout, "rb") as file2:
file1.write(file2.read())
self.chessfile.handle = protoopen(self.chessfile.path)
except SQLAlchemyError as e:
trans.rollback()
print("Importing %s failed! \n%s" % (pgnfile, e))
def do_import(self, filename, info=None, progressbar=None):
self.progressbar = progressbar
orig_filename = filename
count_source = self.conn.execute(
self.count_source.where(source.c.name == orig_filename)).scalar()
if count_source > 0:
print("%s is already imported" % filename)
return
# collect new names not in they dict yet
self.event_data = []
self.site_data = []
self.player_data = []
self.annotator_data = []
self.source_data = []
# collect new games and commit them in big chunks for speed
self.game_data = []
self.tag_game_data = []
if filename.startswith("http"):
filename = download_file(filename, progressbar=progressbar)
if filename is None:
return
else:
if not os.path.isfile(filename):
print("Can't open %s" % filename)
return
if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
with zipfile.ZipFile(filename, "r") as zf:
path = os.path.dirname(filename)
files = [
os.path.join(path, f) for f in zf.namelist()
if f.lower().endswith(".pgn")
]
zf.extractall(path)
else:
files = [filename]
for pgnfile in files:
base_offset = self.chessfile.size if self.append_pgn else 0
basename = os.path.basename(pgnfile)
if progressbar is not None:
GLib.idle_add(progressbar.set_text,
"Reading %s ..." % basename)
else:
print("Reading %s ..." % pgnfile)
size = os.path.getsize(pgnfile)
handle = protoopen(pgnfile)
# estimated game count
all_games = max(size / 840, 1)
handle_json = None
if pgnextractor is not None:
try:
headers_json = os.path.splitext(
pgnfile)[0] + ".headers.json"
if not os.path.isfile(headers_json):
output = subprocess.check_output(
[pgnextractor, "headers", pgnfile]).decode()
for line in output:
if line.startswith("Games"):
all_games = line.split()[1]
handle_json = protoopen(headers_json)
except subprocess.CalledProcessError:
print("pgnextractor failed")
get_id = self.get_id
# use transaction to avoid autocommit slowness
trans = self.conn.begin()
try:
i = 0
for tags in read_games(handle, handle_json):
if not tags:
print("Empty game #%s" % (i + 1))
continue
if self.cancel:
trans.rollback()
return
fenstr = tags["FEN"] if "FEN" in tags else ""
variant = tags["Variant"] if "Variant" in tags else ""
if variant:
if "fischer" in variant.lower() or "960" in variant:
variant = "Fischerandom"
else:
variant = variant.lower().capitalize()
# Fixes for some non statndard Chess960 .pgn
if fenstr and variant == "Fischerandom":
parts = fenstr.split()
parts[0] = parts[0].replace(".", "/").replace("0", "")
if len(parts) == 1:
parts.append("w")
parts.append("-")
parts.append("-")
fenstr = " ".join(parts)
if variant:
if variant not in name2variant:
print("Unknown variant: %s" % variant)
continue
variant = name2variant[variant].variant
if variant == NORMALCHESS:
# lichess uses tag [Variant "Standard"]
variant = 0
else:
variant = 0
white = tags["White"] if "White" in tags else ""
black = tags["Black"] if "Black" in tags else ""
event_id = get_id(tags["Event"] if "Event" in tags else "",
event, EVENT)
site_id = get_id(tags["Site"] if "Site" in tags else "",
site, SITE)
game_date = tags["Date"] if "Date" in tags else ""
try:
if game_date and '?' not in game_date:
ymd = game_date.split('.')
if len(ymd) == 3:
game_year, game_month, game_day = map(int, ymd)
else:
game_year, game_month, game_day = int(
game_date[:4]), None, None
elif game_date and '?' not in game_date[:4]:
game_year, game_month, game_day = int(
game_date[:4]), None, None
else:
game_year, game_month, game_day = None, None, None
except:
game_year, game_month, game_day = None, None, None
game_round = tags['Round'] if "Round" in tags else ""
white_id = get_id(white, player, PLAYER)
black_id = get_id(black, player, PLAYER)
result = tags["Result"] if "Result" in tags else ""
if result in pgn2Const:
result = pgn2Const[result]
else:
if basename != "eco.pgn":
print("Invalid Result tag in game #%s: %s" %
(i + 1, result))
continue
white_elo = tags['WhiteElo'] if "WhiteElo" in tags else "0"
white_elo = int(
white_elo) if white_elo and white_elo.isdigit() else 0
black_elo = tags['BlackElo'] if "BlackElo" in tags else ""
black_elo = int(
black_elo) if black_elo and black_elo.isdigit() else 0
time_control = tags[
"TimeControl"] if "TimeControl" in tags else ""
eco = tags["ECO"][:3] if "ECO" in tags else ""
fen = tags["FEN"] if "FEN" in tags else ""
board_tag = int(tags["Board"]) if "Board" in tags else 0
annotator_id = get_id(
tags["Annotator"] if "Annotator" in tags else "",
annotator, ANNOTATOR)
source_id = get_id(orig_filename,
source,
SOURCE,
info=info)
self.next_id[GAME] += 1
ply_count = tags["PlyCount"] if "PlyCount" in tags else 0
offset = base_offset + int(tags["offset"])
self.game_data.append({
'offset': offset,
'offset8': (offset >> 3) << 3,
'event_id': event_id,
'site_id': site_id,
'date_year': game_year,
'date_month': game_month,
'date_day': game_day,
'round': game_round,
'white_id': white_id,
'black_id': black_id,
'result': result,
'white_elo': white_elo,
'black_elo': black_elo,
'ply_count': ply_count,
'eco': eco,
'fen': fen,
'variant': variant,
'board': board_tag,
'time_control': time_control,
'annotator_id': annotator_id,
'source_id': source_id,
})
i += 1
if len(self.game_data) >= self.CHUNK:
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player,
self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator,
self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source,
self.source_data)
self.source_data = []
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
i / float(all_games))
GLib.idle_add(
progressbar.set_text,
"%s game headers from %s imported" %
(i, basename))
else:
print(pgnfile, i)
if self.event_data:
self.conn.execute(self.ins_event, self.event_data)
self.event_data = []
if self.site_data:
self.conn.execute(self.ins_site, self.site_data)
self.site_data = []
if self.player_data:
self.conn.execute(self.ins_player, self.player_data)
self.player_data = []
if self.annotator_data:
self.conn.execute(self.ins_annotator, self.annotator_data)
self.annotator_data = []
if self.source_data:
self.conn.execute(self.ins_source, self.source_data)
self.source_data = []
if self.game_data:
self.conn.execute(self.ins_game, self.game_data)
self.game_data = []
if progressbar is not None:
GLib.idle_add(progressbar.set_fraction,
i / float(all_games))
GLib.idle_add(
progressbar.set_text,
"%s game headers from %s imported" % (i, basename))
else:
print(pgnfile, i)
trans.commit()
if self.append_pgn:
# reopen database to write
self.db_handle.close()
self.db_handle = protosave(self.chessfile.path,
self.append_pgn)
print("Append from %s to %s" %
(pgnfile, self.chessfile.path))
handle.seek(0)
self.db_handle.writelines(handle)
self.db_handle.close()
handle.close()
if self.chessfile.scoutfish is not None:
# create new .scout from pgnfile we are importing
from pychess.Savers.pgn import scoutfish_path
args = [
scoutfish_path, "make", pgnfile,
"%s" % base_offset
]
output = subprocess.check_output(
args, stderr=subprocess.STDOUT).decode()
# append it to our existing one
if output.find("Processing...done") > 0:
old_scout = self.chessfile.scoutfish.db
new_scout = os.path.splitext(pgnfile)[0] + '.scout'
with open(old_scout,
"ab") as file1, open(new_scout,
"rb") as file2:
file1.write(file2.read())
self.chessfile.handle = protoopen(self.chessfile.path)
except SQLAlchemyError as e:
trans.rollback()
print("Importing %s failed! \n%s" % (pgnfile, e))
