Example #1
0
    def print_db(self):
        a1 = event.alias()
        a2 = site.alias()
        a3 = player.alias()
        a4 = player.alias()

        s = select([
            game.c.id,
            a1.c.name.label('event'),
            a2.c.name.label('site'),
            a3.c.name.label('white'),
            a4.c.name.label('black'), game.c.date_year, game.c.date_month,
            game.c.date_day, game.c.eco, game.c.result, game.c.white_elo,
            game.c.black_elo
        ],
                   and_(game.c.event_id == a1.c.id, game.c.site_id == a2.c.id,
                        game.c.white_id == a3.c.id,
                        game.c.black_id == a4.c.id)).where(
                            and_(a3.c.name.startswith(unicode("Réti")),
                                 a4.c.name.startswith(unicode("Van Nüss"))))

        result = self.conn.execute(s)
        games = result.fetchall()
        for g in games:
            print("%s %s %s %s %s %s %s %s %s %s %s %s" %
                  (g['id'], g['event'], g['site'], g['white'], g['black'],
                   g[5], g[6], g[7], g['eco'], reprResult[g['result']],
                   g['white_elo'], g['black_elo']))
Example #2
0
def ini_tag(engine):
    conn = engine.connect()
    new_values = [
        {"id": LOCAL, "name": unicode("Local game")},
        {"id": ARTIFICIAL, "name": unicode("Chess engine(s)")},
        {"id": REMOTE, "name": unicode("ICS game")},
    ]
    conn.execute(tag.insert(), new_values)
    conn.close()
Example #3
0
def ini_collection():
    conn = engine.connect()
    new_values = [
        {"id": LOCAL, "name": unicode("Local game")},
        {"id": ARTIFICIAL, "name": unicode("Chess engine(s)")},
        {"id": REMOTE, "name": unicode("ICS game")},
        ]
    conn.execute(collection.insert(), new_values)
    conn.close()
Example #4
0
 def __movestr(self, board):
     move = board.lastMove
     if self.fan:           
         movestr = unicode(toFAN(board.prev, move))
     else:          
         movestr =  unicode(toSAN(board.prev, move, True))
     nagsymbols = "".join([nag2symbol(nag) for nag in board.nags])
     # To prevent wrap castling we will use hyphen bullet (U+2043)       
     return "%s%s%s" % (move_count(board), movestr.replace(u'-', u'\u2043'), nagsymbols)
Example #5
0
    def get_id(table, name):
        if not name:
            return None

        selection = select([table.c.id], table.c.name == unicode(name))
        result = conn.execute(selection)
        id_ = result.scalar()
        if id_ is None:
            result = conn.execute(table.insert().values(name=unicode(name)))
            id_ = result.inserted_primary_key[0]
        return id_
Example #6
0
def get_rating_range_display_text (rmin=0, rmax=9999):
    assert isinstance(rmin, type(int())) and rmin >= 0 and rmin <= 9999, rmin
    assert isinstance(rmax, type(int())) and rmax >= 0 and rmax <= 9999, rmax
    if rmin > 0:
        text = "%d" % rmin
        if rmax == 9999:
            text += unicode("↑")
        else:
            text += "-%d" % rmax
    elif rmax != 9999:
        text = unicode("%d↓" % rmax)
    else:
        text = None
    return text
Example #7
0
def get_rating_range_display_text(rmin=0, rmax=9999):
    assert isinstance(rmin, type(int())) and rmin >= 0 and rmin <= 9999, rmin
    assert isinstance(rmax, type(int())) and rmax >= 0 and rmax <= 9999, rmax
    if rmin > 0:
        text = "%d" % rmin
        if rmax == 9999:
            text += unicode("↑")
        else:
            text += "-%d" % rmax
    elif rmax != 9999:
        text = unicode("%d↓" % rmax)
    else:
        text = None
    return text
Example #8
0
    def variation_end(self, iter, index, level, firstboard, parent, opening_node):
        start = iter.get_offset()
        if level == 0:
            self.textbuffer.insert_with_tags_by_name(iter, "]", "variation-toplevel", "variation-margin0")
        elif (level+1) % 2 == 0:
            self.textbuffer.insert_with_tags_by_name(iter, ")", "variation-even", "variation-margin1")
        else:
            self.textbuffer.insert_with_tags_by_name(iter, ")", "variation-uneven", "variation-margin2")

        self.textbuffer.insert_with_tags_by_name(iter, unicode("✖ "), "remove-variation")
        chr = iter.get_char()

        # somehow iter.begins_tag() doesn't work, so we use get_char() instead
        if iter.get_char() != "\n":
            self.textbuffer.insert_with_tags_by_name(iter, "\n", "new_line")

        node = {}
        node["board"] = firstboard
        node["parent"] = parent
        node["vari"] = opening_node
        node["start"] = start
        node["end"] =  iter.get_offset()

        if index == -1:
            self.nodelist.append(node)
        else:
            self.nodelist.insert(index, node)

        return iter.get_offset() - start
Example #9
0
    def variation_end(self, iter, index, level, firstboard, parent, opening_node):
        start = iter.get_offset()
        if level == 0:
            self.textbuffer.insert_with_tags_by_name(iter, "]", "variation-toplevel", "variation-margin0")
        elif (level+1) % 2 == 0:
            self.textbuffer.insert_with_tags_by_name(iter, ")", "variation-even", "variation-margin1")
        else:
            self.textbuffer.insert_with_tags_by_name(iter, ")", "variation-uneven", "variation-margin2")

        self.textbuffer.insert_with_tags_by_name(iter, unicode("✖ "), "remove-variation")
        chr = iter.get_char()

        # somehow iter.begins_tag() doesn't work, so we use get_char() instead
        if iter.get_char() != "\n":
            self.textbuffer.insert_with_tags_by_name(iter, "\n", "new_line")

        node = {}
        node["board"] = firstboard
        node["parent"] = parent
        node["vari"] = opening_node
        node["start"] = start
        node["end"] =  iter.get_offset()

        if index == -1:
            self.nodelist.append(node)
        else:
            self.nodelist.insert(index, node)

        return iter.get_offset() - start
Example #10
0
        def importing():
            drop_indexes(self.gamelist.chessfile.engine)

            self.importer = PgnImport(self.gamelist.chessfile.engine)
            for i, filename in enumerate(filenames):
                filename = unicode(filename)
                GLib.idle_add(self.progressbar0.set_fraction, i / float(len(filenames)))
                # GLib.idle_add(self.progressbar0.set_text, filename)
                if self.importer.cancel:
                    break
                if isinstance(filename, tuple):
                    info_link, pgn_link = filename
                    self.importer.do_import(pgn_link, info=info_link, progressbar=self.progressbar)
                else:
                    self.importer.do_import(filename, progressbar=self.progressbar)

            GLib.idle_add(self.progressbar.set_text, "Recreating indexes...")
            create_indexes(self.gamelist.chessfile.engine)

            self.gamelist.offset = 0
            self.gamelist.chessfile.build_where_tags("")
            self.gamelist.chessfile.build_where_bitboards(0, 0)
            self.gamelist.chessfile.build_query()
            self.gamelist.chessfile.update_count()
            self.gamelist.chessfile.update_count_stats()
            GLib.idle_add(self.gamelist.load_games)
            GLib.idle_add(self.emit, "chessfile_imported", self.gamelist.chessfile)
            GLib.idle_add(self.progress_dialog.hide)
Example #11
0
def toFAN(board, move):
    """ Returns a Figurine Algebraic Notation string of a move """

    san = unicode(toSAN(board, move))
    if board.color == WHITE:
        return san.translate(san2WhiteFanDic)
    else:
        return san.translate(san2BlackFanDic)
Example #12
0
def toFAN (board, move):
    """ Returns a Figurine Algebraic Notation string of a move """
    
    san = unicode(toSAN (board, move))
    if board.color == WHITE:
        return san.translate(san2WhiteFanDic)
    else:
        return san.translate(san2BlackFanDic)
Example #13
0
    def feed(pgnfile, lang):
        cf = load(protoopen(pgnfile))
        rows = []
        old_eco = ""
        ply_max = 0
        for i, game in enumerate(cf.games):
            model = cf.loadToModel(i)

            eco = cf._getTag(i, "ECO")[:3]

            opening = cf._getTag(i, "Opening")
            if opening is None:
                opening = ""

            variation = cf._getTag(i, "Variation")
            if variation is None:
                variation = ""

            base = int(old_eco != eco)

            ply = len(model.moves)
            ply_max = max(ply_max, ply)
            if ply == 0:
                cu = conn.cursor()
                cu.execute(
                    "select * from openings where eco=? and lang='en' and base=1",
                    (eco, ))
                res = cu.fetchone()
                if res is not None:
                    hash = res[0]
            else:
                hash = memoryview(hash_struct.pack(
                    model.boards[-1].board.hash))

            if opening:
                rows.append((hash, base, unicode(eco), unicode(lang),
                             unicode(opening), unicode(variation)))

            old_eco = eco

        c.executemany(
            "insert into openings(hash, base, eco, lang, opening, variation) values (?, ?, ?, ?, ?, ?)",
            rows)
        conn.commit()

        print("Max ply was %s" % ply_max)
Example #14
0
def ellipsize(string, maxlen):
    """ Description: given a string and a length ellipsize will return the string
        if it is smaller than length or it will return the string truncated to length
        and append ... to it

        Return type : str
    """
    if len(string) <= maxlen or maxlen < 4:
        return string
    return string[:maxlen - 1] + unicode("…")
Example #15
0
def ellipsize(string, maxlen):
    """ Description: given a string and a length ellipsize will return the string
        if it is smaller than length or it will return the string truncated to length
        and append ... to it

        Return type : str
    """
    if len(string) <= maxlen or maxlen < 4:
        return string
    return string[:maxlen - 1] + unicode("…")
Example #16
0
 def build_where_tags(self, text):
     if text:
         text = unicode(text)
         self.where_tags = or_(
             pl1.c.name.startswith(text),
             pl2.c.name.startswith(text),
             event.c.name.startswith(text),
             site.c.name.startswith(text),
             annotator.c.name.startswith(text),
         )
     else:
         self.where_tags = None
Example #17
0
    def feed(pgnfile, lang):
        cf = load(protoopen(pgnfile))
        rows = []
        old_eco = ""
        ply_max = 0
        for i, game in enumerate(cf.games):
            model = cf.loadToModel(i)

            eco = cf._getTag(i, "ECO")[:3]
            
            opening = cf._getTag(i, "Opening")
            if opening is None:
                opening = ""

            variation = cf._getTag(i, "Variation")
            if variation is None:
                variation = ""
            
            base = int(old_eco != eco)
            
            ply = len(model.moves)
            ply_max = max(ply_max, ply)
            if ply == 0:
                cu = conn.cursor()
                cu.execute("select * from openings where eco=? and lang='en' and base=1", (eco,))
                res = cu.fetchone()
                if res is not None:
                    hash = res[0]
            else:
                hash = memoryview(hash_struct.pack(model.boards[-1].board.hash))
                
            if opening:
                rows.append((hash, base, unicode(eco), unicode(lang), unicode(opening), unicode(variation)))
                
            old_eco = eco
                
        c.executemany("insert into openings(hash, base, eco, lang, opening, variation) values (?, ?, ?, ?, ?, ?)", rows)
        conn.commit()

        print("Max ply was %s" % ply_max)
Example #18
0
 def newMessage (cls, tag, timestamp, message, importance):
     textview = cls._getPageFromTag(tag)["textview"]
     
     if not tag in cls.tagToTime or timestamp-cls.tagToTime[tag] >= 1:
         t = time.strftime("%H:%M:%S", time.localtime(timestamp))
         textview.get_buffer().insert_with_tags_by_name(
             textview.get_buffer().get_end_iter(), "\n%s\n%s\n"%(t,"-"*60), str(logging.INFO))
         cls.tagToTime[tag] = timestamp
     
     if isinstance(message, str):
         message = unicode(message, "utf-8", 'rawreplace')
     if not message.endswith("\n"):
         message = "%s\n" % message
     textview.get_buffer().insert_with_tags_by_name(
         textview.get_buffer().get_end_iter(), message, str(importance))
Example #19
0
    def print_db(self):
        a1 = event.alias()
        a2 = site.alias()
        a3 = player.alias()
        a4 = player.alias()

        s = select(
            [game.c.id, a1.c.name.label('event'), a2.c.name.label('site'),
             a3.c.name.label('white'), a4.c.name.label('black'),
             game.c.date_year, game.c.date_month, game.c.date_day, game.c.eco,
             game.c.result, game.c.white_elo, game.c.black_elo],
            and_(game.c.event_id == a1.c.id, game.c.site_id == a2.c.id,
                 game.c.white_id == a3.c.id,
                 game.c.black_id == a4.c.id)).where(and_(
                     a3.c.name.startswith(unicode(
                         "Réti")), a4.c.name.startswith(unicode("Van Nüss"))))

        result = self.conn.execute(s)
        games = result.fetchall()
        for g in games:
            print("%s %s %s %s %s %s %s %s %s %s %s %s" %
                  (g['id'], g['event'], g['site'], g['white'], g['black'],
                   g[5], g[6], g[7], g['eco'], reprResult[g['result']],
                   g['white_elo'], g['black_elo']))
Example #20
0
    def newMessage(cls, tag, timestamp, message, importance):
        textview = cls._getPageFromTag(tag)["textview"]

        if not tag in cls.tagToTime or timestamp - cls.tagToTime[tag] >= 1:
            t = time.strftime("%H:%M:%S", time.localtime(timestamp))
            textview.get_buffer().insert_with_tags_by_name(
                textview.get_buffer().get_end_iter(),
                "\n%s\n%s\n" % (t, "-" * 60), str(logging.INFO))
            cls.tagToTime[tag] = timestamp

        if isinstance(message, str):
            message = unicode(message, "utf-8", 'rawreplace')
        if not message.endswith("\n"):
            message = "%s\n" % message
        textview.get_buffer().insert_with_tags_by_name(
            textview.get_buffer().get_end_iter(), message, str(importance))
Example #21
0
        def importing():
            drop_indexes(self.gamelist.chessfile.engine)

            self.importer = PgnImport(self.gamelist.chessfile, append_pgn=True)
            for i, filename in enumerate(filenames):
                GLib.idle_add(self.progressbar0.set_fraction, i / float(len(filenames)))
                # GLib.idle_add(self.progressbar0.set_text, filename)
                if self.importer.cancel:
                    break
                if isinstance(filename, tuple):
                    info_link, pgn_link = filename
                    self.importer.do_import(pgn_link, info=info_link, progressbar=self.progressbar1)
                else:
                    filename = unicode(filename)
                    self.importer.do_import(filename, progressbar=self.progressbar1)

            GLib.idle_add(self.progressbar1.set_text, "Recreating indexes...")

            # .sqlite
            create_indexes(self.gamelist.chessfile.engine)

            # .scout
            self.gamelist.chessfile.init_scoutfish()
            if self.gamelist.chessfile.scoutfish is not None:
                self.gamelist.chessfile.scoutfish.make()

            # .bin
            self.gamelist.chessfile.init_chess_db()
            if self.gamelist.chessfile.chess_db is not None:
                self.gamelist.chessfile.chess_db.make()

            self.gamelist.chessfile.set_tags_filter("")
            self.gamelist.chessfile.set_fen_filter(FEN_START)
            self.gamelist.chessfile.set_scout_filter("")
            GLib.idle_add(self.gamelist.load_games)
            GLib.idle_add(self.emit, "chessfile_imported", self.gamelist.chessfile)
            GLib.idle_add(self.progress_dialog.hide)
Example #22
0
    def do_import(self, filename, info=None, progressbar=None):
        DB_MAXINT_SHIFT = get_maxint_shift(self.engine)
        self.progressbar = progressbar

        orig_filename = filename
        count_source = self.conn.execute(
            self.count_source.where(source.c.name == orig_filename)).scalar()
        if count_source > 0:
            print("%s is already imported" % filename)
            return

        # collect new names not in they dict yet
        self.event_data = []
        self.site_data = []
        self.player_data = []
        self.annotator_data = []
        self.source_data = []

        # collect new games and commit them in big chunks for speed
        self.game_data = []
        self.bitboard_data = []
        self.stat_ins_data = []
        self.stat_upd_data = []
        self.tag_game_data = []

        if filename.startswith("http"):
            filename = download_file(filename, progressbar=progressbar)
            if filename is None:
                return
        else:
            if not os.path.isfile(filename):
                print("Can't open %s" % filename)
                return

        if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
            zf = zipfile.ZipFile(filename, "r")
            files = [f for f in zf.namelist() if f.lower().endswith(".pgn")]
        else:
            zf = None
            files = [filename]

        for pgnfile in files:
            basename = os.path.basename(pgnfile)
            if progressbar is not None:
                GLib.idle_add(progressbar.set_text,
                              "Reading %s ..." % basename)
            else:
                print("Reading %s ..." % pgnfile)

            if zf is None:
                size = os.path.getsize(pgnfile)
                handle = protoopen(pgnfile)
            else:
                size = zf.getinfo(pgnfile).file_size
                handle = io.TextIOWrapper(zf.open(pgnfile),
                                          encoding=PGN_ENCODING,
                                          newline='')

            cf = PgnBase(handle, [])

            # estimated game count
            all_games = max(size / 840, 1)
            self.CHUNK = 1000 if all_games > 5000 else 100

            get_id = self.get_id
            # use transaction to avoid autocommit slowness
            trans = self.conn.begin()
            try:
                i = 0
                for tagtext, movetext in read_games(handle):
                    tags = defaultdict(str, tagre.findall(tagtext))
                    if not tags:
                        print("Empty game #%s" % (i + 1))
                        continue

                    if self.cancel:
                        trans.rollback()
                        return

                    fenstr = tags.get("FEN")

                    variant = tags.get("Variant")
                    if variant:
                        if "fischer" in variant.lower() or "960" in variant:
                            variant = "Fischerandom"
                        else:
                            variant = variant.lower().capitalize()

                    # Fixes for some non statndard Chess960 .pgn
                    if fenstr and variant == "Fischerandom":
                        parts = fenstr.split()
                        parts[0] = parts[0].replace(".", "/").replace("0", "")
                        if len(parts) == 1:
                            parts.append("w")
                            parts.append("-")
                            parts.append("-")
                        fenstr = " ".join(parts)

                    if variant:
                        if variant not in name2variant:
                            print("Unknown variant: %s" % variant)
                            continue
                        variant = name2variant[variant].variant
                        if variant == NORMALCHESS:
                            # lichess uses tag [Variant "Standard"]
                            variant = 0
                            board = START_BOARD.clone()
                        else:
                            board = LBoard(variant)
                    elif fenstr:
                        variant = 0
                        board = LBoard()
                    else:
                        variant = 0
                        board = START_BOARD.clone()

                    if fenstr:
                        try:
                            board.applyFen(fenstr)
                        except SyntaxError as e:
                            print(
                                _("The game #%s can't be loaded, because of an error parsing FEN"
                                  ) % (i + 1), e.args[0])
                            continue
                    elif variant:
                        board.applyFen(FEN_START)

                    movelist = array("H")
                    comments = []
                    cf.error = None

                    # First we try to use simple_parse_movetext()
                    # assuming most games in .pgn contains only moves
                    # without any comments/variations
                    simple = False
                    if not fenstr and not variant:
                        bitboards = []
                        simple = cf.simple_parse_movetext(
                            movetext, board, movelist, bitboards)

                        if cf.error is not None:
                            print("ERROR in %s game #%s" % (pgnfile, i + 1),
                                  cf.error.args[0])
                            continue

                    # If simple_parse_movetext() find any comments/variations
                    # we restart parsing with full featured parse_movetext()
                    if not simple:
                        movelist = array("H")
                        bitboards = None

                        # in case simple_parse_movetext failed we have to reset our lboard
                        if not fenstr and not variant:
                            board = START_BOARD.clone()

                        # parse movetext to create boards tree structure
                        boards = [board]
                        boards = cf.parse_movetext(movetext,
                                                   boards[0],
                                                   -1,
                                                   pgn_import=True)

                        if cf.error is not None:
                            print("ERROR in %s game #%s" % (pgnfile, i + 1),
                                  cf.error.args[0])
                            continue

                        # create movelist and comments from boards tree
                        walk(boards[0], movelist, comments)

                    white = tags.get('White')
                    black = tags.get('Black')

                    if not movelist:
                        if (not comments) and (not white) and (not black):
                            print("Empty game #%s" % (i + 1))
                            continue

                    event_id = get_id(tags.get('Event'), event, EVENT)

                    site_id = get_id(tags.get('Site'), site, SITE)

                    game_date = tags.get('Date').strip()
                    try:
                        if game_date and '?' not in game_date:
                            ymd = game_date.split('.')
                            if len(ymd) == 3:
                                game_year, game_month, game_day = map(int, ymd)
                            else:
                                game_year, game_month, game_day = int(
                                    game_date[:4]), None, None
                        elif game_date and '?' not in game_date[:4]:
                            game_year, game_month, game_day = int(
                                game_date[:4]), None, None
                        else:
                            game_year, game_month, game_day = None, None, None
                    except:
                        game_year, game_month, game_day = None, None, None

                    game_round = tags.get('Round')

                    white_fide_id = tags.get('WhiteFideId')
                    black_fide_id = tags.get('BlackFideId')

                    white_id = get_id(unicode(white),
                                      player,
                                      PLAYER,
                                      fide_id=white_fide_id)
                    black_id = get_id(unicode(black),
                                      player,
                                      PLAYER,
                                      fide_id=black_fide_id)

                    result = tags.get("Result")
                    if result in pgn2Const:
                        result = pgn2Const[result]
                    else:
                        print("Invalid Result tag in game #%s: %s" %
                              (i + 1, result))
                        continue

                    white_elo = tags.get('WhiteElo')
                    white_elo = int(
                        white_elo
                    ) if white_elo and white_elo.isdigit() else None

                    black_elo = tags.get('BlackElo')
                    black_elo = int(
                        black_elo
                    ) if black_elo and black_elo.isdigit() else None

                    time_control = tags.get("TimeControl")

                    eco = tags.get("ECO")
                    eco = eco[:3] if eco else None

                    fen = tags.get("FEN")

                    board_tag = tags.get("Board")

                    annotator_id = get_id(tags.get("Annotator"), annotator,
                                          ANNOTATOR)

                    source_id = get_id(unicode(orig_filename),
                                       source,
                                       SOURCE,
                                       info=info)

                    game_id = self.next_id[GAME]
                    self.next_id[GAME] += 1

                    # annotated game
                    if bitboards is None:
                        for ply, board in enumerate(boards):
                            if ply == 0:
                                continue
                            bb = board.friends[0] | board.friends[1]
                            # Avoid to include mate in x .pgn collections and similar in opening tree
                            if fen and "/pppppppp/8/8/8/8/PPPPPPPP/" not in fen:
                                ply = -1
                            self.bitboard_data.append({
                                'game_id':
                                game_id,
                                'ply':
                                ply,
                                'bitboard':
                                bb - DB_MAXINT_SHIFT,
                            })

                            if ply <= STAT_PLY_MAX:
                                self.stat_ins_data.append({
                                    'ply': ply,
                                    'bitboard': bb - DB_MAXINT_SHIFT,
                                    'count': 0,
                                    'whitewon': 0,
                                    'blackwon': 0,
                                    'draw': 0,
                                    'white_elo_count': 0,
                                    'black_elo_count': 0,
                                    'white_elo': 0,
                                    'black_elo': 0,
                                })
                                self.stat_upd_data.append({
                                    '_ply':
                                    ply,
                                    '_bitboard':
                                    bb - DB_MAXINT_SHIFT,
                                    '_count':
                                    1,
                                    '_whitewon':
                                    1 if result == WHITEWON else 0,
                                    '_blackwon':
                                    1 if result == BLACKWON else 0,
                                    '_draw':
                                    1 if result == DRAW else 0,
                                    '_white_elo_count':
                                    1 if white_elo is not None else 0,
                                    '_black_elo_count':
                                    1 if black_elo is not None else 0,
                                    '_white_elo':
                                    white_elo if white_elo is not None else 0,
                                    '_black_elo':
                                    black_elo if black_elo is not None else 0,
                                })

                    # simple game
                    else:
                        for ply, bb in enumerate(bitboards):
                            if ply == 0:
                                continue
                            self.bitboard_data.append({
                                'game_id':
                                game_id,
                                'ply':
                                ply,
                                'bitboard':
                                bb - DB_MAXINT_SHIFT,
                            })

                            if ply <= STAT_PLY_MAX:
                                self.stat_ins_data.append({
                                    'ply': ply,
                                    'bitboard': bb - DB_MAXINT_SHIFT,
                                    'count': 0,
                                    'whitewon': 0,
                                    'blackwon': 0,
                                    'draw': 0,
                                    'white_elo_count': 0,
                                    'black_elo_count': 0,
                                    'white_elo': 0,
                                    'black_elo': 0,
                                })
                                self.stat_upd_data.append({
                                    '_ply':
                                    ply,
                                    '_bitboard':
                                    bb - DB_MAXINT_SHIFT,
                                    '_count':
                                    1,
                                    '_whitewon':
                                    1 if result == WHITEWON else 0,
                                    '_blackwon':
                                    1 if result == BLACKWON else 0,
                                    '_draw':
                                    1 if result == DRAW else 0,
                                    '_white_elo_count':
                                    1 if white_elo is not None else 0,
                                    '_black_elo_count':
                                    1 if black_elo is not None else 0,
                                    '_white_elo':
                                    white_elo if white_elo is not None else 0,
                                    '_black_elo':
                                    black_elo if black_elo is not None else 0,
                                })

                    ply_count = tags.get("PlyCount")
                    if not ply_count and not fen:
                        ply_count = len(
                            bitboards) if bitboards is not None else len(
                                boards)

                    self.game_data.append({
                        'event_id':
                        event_id,
                        'site_id':
                        site_id,
                        'date_year':
                        game_year,
                        'date_month':
                        game_month,
                        'date_day':
                        game_day,
                        'round':
                        game_round,
                        'white_id':
                        white_id,
                        'black_id':
                        black_id,
                        'result':
                        result,
                        'white_elo':
                        white_elo,
                        'black_elo':
                        black_elo,
                        'ply_count':
                        ply_count,
                        'eco':
                        eco,
                        'fen':
                        fen,
                        'variant':
                        variant,
                        'board':
                        board_tag,
                        'time_control':
                        time_control,
                        'annotator_id':
                        annotator_id,
                        'source_id':
                        source_id,
                        'movelist':
                        movelist.tostring(),
                        'comments':
                        unicode("|".join(comments)),
                    })

                    i += 1

                    if len(self.game_data) >= self.CHUNK:
                        if self.event_data:
                            self.conn.execute(self.ins_event, self.event_data)
                            self.event_data = []

                        if self.site_data:
                            self.conn.execute(self.ins_site, self.site_data)
                            self.site_data = []

                        if self.player_data:
                            self.conn.execute(self.ins_player,
                                              self.player_data)
                            self.player_data = []

                        if self.annotator_data:
                            self.conn.execute(self.ins_annotator,
                                              self.annotator_data)
                            self.annotator_data = []

                        if self.source_data:
                            self.conn.execute(self.ins_source,
                                              self.source_data)
                            self.source_data = []

                        self.conn.execute(self.ins_game, self.game_data)
                        self.game_data = []

                        if self.bitboard_data:
                            self.conn.execute(self.ins_bitboard,
                                              self.bitboard_data)
                            self.bitboard_data = []

                            self.conn.execute(self.ins_stat,
                                              self.stat_ins_data)
                            self.conn.execute(self.upd_stat,
                                              self.stat_upd_data)
                            self.stat_ins_data = []
                            self.stat_upd_data = []

                        if progressbar is not None:
                            GLib.idle_add(progressbar.set_fraction,
                                          i / float(all_games))
                            GLib.idle_add(
                                progressbar.set_text,
                                "%s games from %s imported" % (i, basename))
                        else:
                            print(pgnfile, i)

                if self.event_data:
                    self.conn.execute(self.ins_event, self.event_data)
                    self.event_data = []

                if self.site_data:
                    self.conn.execute(self.ins_site, self.site_data)
                    self.site_data = []

                if self.player_data:
                    self.conn.execute(self.ins_player, self.player_data)
                    self.player_data = []

                if self.annotator_data:
                    self.conn.execute(self.ins_annotator, self.annotator_data)
                    self.annotator_data = []

                if self.source_data:
                    self.conn.execute(self.ins_source, self.source_data)
                    self.source_data = []

                if self.game_data:
                    self.conn.execute(self.ins_game, self.game_data)
                    self.game_data = []

                if self.bitboard_data:
                    self.conn.execute(self.ins_bitboard, self.bitboard_data)
                    self.bitboard_data = []

                    self.conn.execute(self.ins_stat, self.stat_ins_data)
                    self.conn.execute(self.upd_stat, self.stat_upd_data)
                    self.stat_ins_data = []
                    self.stat_upd_data = []

                if progressbar is not None:
                    GLib.idle_add(progressbar.set_fraction,
                                  i / float(all_games))
                    GLib.idle_add(progressbar.set_text,
                                  "%s games from %s imported" % (i, basename))
                else:
                    print(pgnfile, i)
                trans.commit()

            except SQLAlchemyError as e:
                trans.rollback()
                print("Importing %s failed! \n%s" % (pgnfile, e))
Example #23
0
    def do_import(self, filename):
        print(filename)
        # collect new names not in they dict yet
        self.collection_data = []
        self.event_data = []
        self.site_data = []
        self.player_data = []
        self.annotator_data = []
        
        # collect new games and commit them in big chunks for speed
        self.game_data = []

        if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
            zf = zipfile.ZipFile(filename, "r")
            files = [f for f in zf.namelist() if f.lower().endswith(".pgn")]
        else:
            zf = None
            files = [filename]
        
        for pgnfile in files:
            if zf is None:
                cf = pgn_load(open(pgnfile, "rU"))
            else:
                cf = pgn_load(zf.open(pgnfile, "rU"))
             
            # use transaction to avoid autocommit slowness
            trans = self.conn.begin()
            try:
                for i, game in enumerate(cf.games):
                    #print i+1#, cf.get_player_names(i)
                    movelist = array("H")
                    comments = []
                    cf.error = None

                    fenstr = cf._getTag(i, "FEN")
                    variant = cf.get_variant(i)

                    # Fixes for some non statndard Chess960 .pgn
                    if variant==0 and (fenstr is not None) and "Chess960" in cf._getTag(i,"Event"):
                        cf.tagcache[i]["Variant"] = "Fischerandom"
                        variant = 1
                        parts = fenstr.split()
                        parts[0] = parts[0].replace(".", "/").replace("0", "")
                        if len(parts) == 1:
                            parts.append("w")
                            parts.append("-")
                            parts.append("-")
                        fenstr = " ".join(parts)
                    
                    if variant:
                        board = LBoard(FISCHERRANDOMCHESS)
                    else:
                        board = LBoard()

                    if fenstr:
                        try:
                            board.applyFen(fenstr)
                        except SyntaxError as e:
                            print(_("The game #%s can't be loaded, because of an error parsing FEN") % (i+1), e.args[0])
                            continue
                    else:
                        board = LBoard_FEN_START.clone()

                    boards = [board]
                    movetext = cf.get_movetext(i)
                    boards = cf.parse_string(movetext, boards[0], -1)

                    if cf.error is not None:
                        print("ERROR in game #%s" % (i+1), cf.error.args[0])
                        continue

                    walk(boards[0], movelist, comments)
                    
                    if not movelist:
                        if (not comments) and (cf._getTag(i, 'White') is None) and (cf._getTag(i, 'Black') is None):
                            print("empty game")
                            continue
                    
                    event_id = self.get_id(cf._getTag(i, 'Event'), event, EVENT)

                    site_id = self.get_id(cf._getTag(i, 'Site'), site, SITE)

                    game_date = cf._getTag(i, 'Date')
                    if game_date and not '?' in game_date:
                        ymd = game_date.split('.')
                        if len(ymd) == 3:
                            game_year, game_month, game_day = map(int, ymd)
                        else:
                            game_year, game_month, game_day = int(game_date[:4]), None, None
                    elif game_date and not '?' in game_date[:4]:
                        game_year, game_month, game_day = int(game_date[:4]), None, None
                    else:
                        game_year, game_month, game_day = None, None, None

                    game_round = cf._getTag(i, 'Round')

                    white, black = cf.get_player_names(i)
                    white_id = self.get_id(white, player, PLAYER)
                    black_id = self.get_id(black, player, PLAYER)

                    result = cf.get_result(i)
     
                    white_elo = cf._getTag(i, 'WhiteElo')
                    white_elo = int(white_elo) if white_elo and white_elo.isdigit() else None
                    
                    black_elo = cf._getTag(i, 'BlackElo')
                    black_elo = int(black_elo) if black_elo and black_elo.isdigit() else None
     
                    ply_count = cf._getTag(i, "PlyCount")
     
                    event_date = cf._getTag(i, 'EventDate')
     
                    eco = cf._getTag(i, "ECO")
                    eco = eco[:3] if eco else None

                    fen = cf._getTag(i, "FEN")
     
                    variant = cf.get_variant(i)
                    
                    board = cf._getTag(i, "Board")
                    
                    annotator = cf._getTag(i, "Annotator")
                    annotator_id = self.get_id(annotator, annotator, ANNOTATOR)

                    collection_id = self.get_id(unicode(pgnfile), collection, COLLECTION)

                    self.game_data.append({
                        'event_id': event_id,
                        'site_id': site_id,
                        'date_year': game_year,
                        'date_month': game_month,
                        'date_day': game_day,
                        'round': game_round,
                        'white_id': white_id,
                        'black_id': black_id,
                        'result': result,
                        'white_elo': white_elo,
                        'black_elo': black_elo,
                        'ply_count': ply_count,
                        'eco': eco,
                        'fen': fen,
                        'variant': variant,
                        'board': board,
                        'annotator_id': annotator_id,
                        'collection_id': collection_id,
                        'movelist': movelist.tostring(),
                        'comments': unicode("|".join(comments)),
                        })

                    if len(self.game_data) >= CHUNK:
                        if self.collection_data:
                            self.conn.execute(self.ins_collection, self.collection_data)
                            self.collection_data = []

                        if self.event_data:
                            self.conn.execute(self.ins_event, self.event_data)
                            self.event_data = []

                        if self.site_data:
                            self.conn.execute(self.ins_site, self.site_data)
                            self.site_data = []

                        if self.player_data:
                            self.conn.execute(self.ins_player, self.player_data)
                            self.player_data = []

                        if self.annotator_data:
                            self.conn.execute(self.ins_annotator, self.annotator_data)
                            self.annotator_data = []

                        self.conn.execute(self.ins_game, self.game_data)
                        self.game_data = []
                        print(pgnfile, i+1)
                    
                if self.collection_data:
                    self.conn.execute(self.ins_collection, self.collection_data)
                    self.collection_data = []

                if self.event_data:
                    self.conn.execute(self.ins_event, self.event_data)
                    self.event_data = []

                if self.site_data:
                    self.conn.execute(self.ins_site, self.site_data)
                    self.site_data = []

                if self.player_data:
                    self.conn.execute(self.ins_player, self.player_data)
                    self.player_data = []

                if self.annotator_data:
                    self.conn.execute(self.ins_annotator, self.annotator_data)
                    self.annotator_data = []

                if self.game_data:
                    self.conn.execute(self.ins_game, self.game_data)
                    self.game_data = []

                print(pgnfile, i+1)
                trans.commit()

            except ProgrammingError as e:
                trans.rollback()
                print("Importing %s failed! %s" % (file, e))
Example #24
0
from pychess.compat import unicode
from pychess.Utils.const import *
from pychess.Utils.lutils.LBoard import LBoard
from pychess.Savers.ChessFile import LoadingError
from pychess.Savers.pgnbase import pgn_load
from pychess.Database.dbwalk import walk
from pychess.Database.model import engine, metadata, collection, event,\
                            site, player, game, annotator, ini_collection

CHUNK = 1000

EVENT, SITE, PLAYER, ANNOTATOR, COLLECTION = range(5)

removeDic = {
    ord(unicode("'")): None,
    ord(unicode(",")): None,
    ord(unicode(".")): None,
    ord(unicode("-")): None,
    ord(unicode(" ")): None,
}

LBoard_FEN_START = LBoard()
LBoard_FEN_START.applyFen(FEN_START)

class PgnImport():
    def __init__(self):
        self.conn = engine.connect()
        
        self.ins_collection = collection.insert()
        self.ins_event = event.insert()
Example #25
0
def ellipsize (string, maxlen):
    if len(string) <= maxlen or maxlen < 4:
        return string
    return string[:maxlen-1] + unicode("…")
Example #26
0
from pychess.compat import unicode
from pychess.Utils.const import *
from pychess.Utils.lutils.LBoard import LBoard
from pychess.Savers.ChessFile import LoadingError
from pychess.Savers.pgnbase import pgn_load
from pychess.Database.dbwalk import walk
from pychess.Database.model import engine, metadata, collection, event,\
                            site, player, game, annotator, ini_collection

CHUNK = 1000

EVENT, SITE, PLAYER, ANNOTATOR, COLLECTION = range(5)

removeDic = {
    ord(unicode("'")): None,
    ord(unicode(",")): None,
    ord(unicode(".")): None,
    ord(unicode("-")): None,
    ord(unicode(" ")): None,
}

LBoard_FEN_START = LBoard()
LBoard_FEN_START.applyFen(FEN_START)


class PgnImport():
    def __init__(self):
        self.conn = engine.connect()

        self.ins_collection = collection.insert()
Example #27
0
    def do_import(self, filename):
        print(filename)
        # collect new names not in they dict yet
        self.collection_data = []
        self.event_data = []
        self.site_data = []
        self.player_data = []
        self.annotator_data = []

        # collect new games and commit them in big chunks for speed
        self.game_data = []

        if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
            zf = zipfile.ZipFile(filename, "r")
            files = [f for f in zf.namelist() if f.lower().endswith(".pgn")]
        else:
            zf = None
            files = [filename]

        for pgnfile in files:
            if zf is None:
                cf = pgn_load(open(pgnfile, "rU"))
            else:
                cf = pgn_load(zf.open(pgnfile, "rU"))

            # use transaction to avoid autocommit slowness
            trans = self.conn.begin()
            try:
                for i, game in enumerate(cf.games):
                    #print i+1#, cf.get_player_names(i)
                    movelist = array("H")
                    comments = []
                    cf.error = None

                    fenstr = cf._getTag(i, "FEN")
                    variant = cf.get_variant(i)

                    # Fixes for some non statndard Chess960 .pgn
                    if variant == 0 and (
                            fenstr is not None) and "Chess960" in cf._getTag(
                                i, "Event"):
                        cf.tagcache[i]["Variant"] = "Fischerandom"
                        variant = 1
                        parts = fenstr.split()
                        parts[0] = parts[0].replace(".", "/").replace("0", "")
                        if len(parts) == 1:
                            parts.append("w")
                            parts.append("-")
                            parts.append("-")
                        fenstr = " ".join(parts)

                    if variant:
                        board = LBoard(FISCHERRANDOMCHESS)
                    else:
                        board = LBoard()

                    if fenstr:
                        try:
                            board.applyFen(fenstr)
                        except SyntaxError as e:
                            print(
                                _("The game #%s can't be loaded, because of an error parsing FEN"
                                  ) % (i + 1), e.args[0])
                            continue
                    else:
                        board = LBoard_FEN_START.clone()

                    boards = [board]
                    movetext = cf.get_movetext(i)
                    boards = cf.parse_string(movetext, boards[0], -1)

                    if cf.error is not None:
                        print("ERROR in game #%s" % (i + 1), cf.error.args[0])
                        continue

                    walk(boards[0], movelist, comments)

                    if not movelist:
                        if (not comments) and (cf._getTag(i, 'White') is
                                               None) and (cf._getTag(
                                                   i, 'Black') is None):
                            print("empty game")
                            continue

                    event_id = self.get_id(cf._getTag(i, 'Event'), event,
                                           EVENT)

                    site_id = self.get_id(cf._getTag(i, 'Site'), site, SITE)

                    game_date = cf._getTag(i, 'Date')
                    if game_date and not '?' in game_date:
                        ymd = game_date.split('.')
                        if len(ymd) == 3:
                            game_year, game_month, game_day = map(int, ymd)
                        else:
                            game_year, game_month, game_day = int(
                                game_date[:4]), None, None
                    elif game_date and not '?' in game_date[:4]:
                        game_year, game_month, game_day = int(
                            game_date[:4]), None, None
                    else:
                        game_year, game_month, game_day = None, None, None

                    game_round = cf._getTag(i, 'Round')

                    white, black = cf.get_player_names(i)
                    white_id = self.get_id(white, player, PLAYER)
                    black_id = self.get_id(black, player, PLAYER)

                    result = cf.get_result(i)

                    white_elo = cf._getTag(i, 'WhiteElo')
                    white_elo = int(
                        white_elo
                    ) if white_elo and white_elo.isdigit() else None

                    black_elo = cf._getTag(i, 'BlackElo')
                    black_elo = int(
                        black_elo
                    ) if black_elo and black_elo.isdigit() else None

                    ply_count = cf._getTag(i, "PlyCount")

                    event_date = cf._getTag(i, 'EventDate')

                    eco = cf._getTag(i, "ECO")
                    eco = eco[:3] if eco else None

                    fen = cf._getTag(i, "FEN")

                    variant = cf.get_variant(i)

                    board = cf._getTag(i, "Board")

                    annotator = cf._getTag(i, "Annotator")
                    annotator_id = self.get_id(annotator, annotator, ANNOTATOR)

                    collection_id = self.get_id(unicode(pgnfile), collection,
                                                COLLECTION)

                    self.game_data.append({
                        'event_id':
                        event_id,
                        'site_id':
                        site_id,
                        'date_year':
                        game_year,
                        'date_month':
                        game_month,
                        'date_day':
                        game_day,
                        'round':
                        game_round,
                        'white_id':
                        white_id,
                        'black_id':
                        black_id,
                        'result':
                        result,
                        'white_elo':
                        white_elo,
                        'black_elo':
                        black_elo,
                        'ply_count':
                        ply_count,
                        'eco':
                        eco,
                        'fen':
                        fen,
                        'variant':
                        variant,
                        'board':
                        board,
                        'annotator_id':
                        annotator_id,
                        'collection_id':
                        collection_id,
                        'movelist':
                        movelist.tostring(),
                        'comments':
                        unicode("|".join(comments)),
                    })

                    if len(self.game_data) >= CHUNK:
                        if self.collection_data:
                            self.conn.execute(self.ins_collection,
                                              self.collection_data)
                            self.collection_data = []

                        if self.event_data:
                            self.conn.execute(self.ins_event, self.event_data)
                            self.event_data = []

                        if self.site_data:
                            self.conn.execute(self.ins_site, self.site_data)
                            self.site_data = []

                        if self.player_data:
                            self.conn.execute(self.ins_player,
                                              self.player_data)
                            self.player_data = []

                        if self.annotator_data:
                            self.conn.execute(self.ins_annotator,
                                              self.annotator_data)
                            self.annotator_data = []

                        self.conn.execute(self.ins_game, self.game_data)
                        self.game_data = []
                        print(pgnfile, i + 1)

                if self.collection_data:
                    self.conn.execute(self.ins_collection,
                                      self.collection_data)
                    self.collection_data = []

                if self.event_data:
                    self.conn.execute(self.ins_event, self.event_data)
                    self.event_data = []

                if self.site_data:
                    self.conn.execute(self.ins_site, self.site_data)
                    self.site_data = []

                if self.player_data:
                    self.conn.execute(self.ins_player, self.player_data)
                    self.player_data = []

                if self.annotator_data:
                    self.conn.execute(self.ins_annotator, self.annotator_data)
                    self.annotator_data = []

                if self.game_data:
                    self.conn.execute(self.ins_game, self.game_data)
                    self.game_data = []

                print(pgnfile, i + 1)
                trans.commit()

            except ProgrammingError as e:
                trans.rollback()
                print("Importing %s failed! %s" % (file, e))
Example #28
0
    def do_import(self, filename, info=None, progressbar=None):
        self.progressbar = progressbar

        orig_filename = filename
        count_source = self.conn.execute(
            self.count_source.where(source.c.name == orig_filename)).scalar()
        if count_source > 0:
            print("%s is already imported" % filename)
            return

        # collect new names not in they dict yet
        self.event_data = []
        self.site_data = []
        self.player_data = []
        self.annotator_data = []
        self.source_data = []

        # collect new games and commit them in big chunks for speed
        self.game_data = []
        self.tag_game_data = []

        if filename.startswith("http"):
            filename = download_file(filename, progressbar=progressbar)
            if filename is None:
                return
        else:
            if not os.path.isfile(filename):
                print("Can't open %s" % filename)
                return

        if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
            with zipfile.ZipFile(filename, "r") as zf:
                path = os.path.dirname(filename)
                files = [
                    os.path.join(path, f) for f in zf.namelist()
                    if f.lower().endswith(".pgn")
                ]
                zf.extractall(path)
        else:
            files = [filename]

        for pgnfile in files:
            base_offset = self.chessfile.size if self.append_pgn else 0

            basename = os.path.basename(pgnfile)
            if progressbar is not None:
                GLib.idle_add(progressbar.set_text,
                              "Reading %s ..." % basename)
            else:
                print("Reading %s ..." % pgnfile)

            size = os.path.getsize(pgnfile)
            handle = protoopen(pgnfile)

            # estimated game count
            all_games = max(size / 840, 1)

            get_id = self.get_id
            # use transaction to avoid autocommit slowness
            trans = self.conn.begin()
            try:
                i = 0
                for offs, tags in read_games(handle):
                    if not tags:
                        print("Empty game #%s" % (i + 1))
                        continue

                    if self.cancel:
                        trans.rollback()
                        return

                    fenstr = tags.get("FEN")

                    variant = tags.get("Variant")
                    if variant:
                        if "fischer" in variant.lower() or "960" in variant:
                            variant = "Fischerandom"
                        else:
                            variant = variant.lower().capitalize()

                    # Fixes for some non statndard Chess960 .pgn
                    if fenstr and variant == "Fischerandom":
                        parts = fenstr.split()
                        parts[0] = parts[0].replace(".", "/").replace("0", "")
                        if len(parts) == 1:
                            parts.append("w")
                            parts.append("-")
                            parts.append("-")
                        fenstr = " ".join(parts)

                    if variant:
                        if variant not in name2variant:
                            print("Unknown variant: %s" % variant)
                            continue
                        variant = name2variant[variant].variant
                        if variant == NORMALCHESS:
                            # lichess uses tag [Variant "Standard"]
                            variant = 0
                    else:
                        variant = 0

                    white = tags.get('White')
                    black = tags.get('Black')

                    event_id = get_id(tags.get('Event'), event, EVENT)

                    site_id = get_id(tags.get('Site'), site, SITE)

                    game_date = tags['Date'].strip()
                    try:
                        if game_date and '?' not in game_date:
                            ymd = game_date.split('.')
                            if len(ymd) == 3:
                                game_year, game_month, game_day = map(int, ymd)
                            else:
                                game_year, game_month, game_day = int(
                                    game_date[:4]), None, None
                        elif game_date and '?' not in game_date[:4]:
                            game_year, game_month, game_day = int(
                                game_date[:4]), None, None
                        else:
                            game_year, game_month, game_day = None, None, None
                    except:
                        game_year, game_month, game_day = None, None, None

                    game_round = tags.get('Round')

                    white_id = get_id(unicode(white), player, PLAYER)
                    black_id = get_id(unicode(black), player, PLAYER)

                    result = tags.get("Result")
                    if result in pgn2Const:
                        result = pgn2Const[result]
                    else:
                        print("Invalid Result tag in game #%s: %s" %
                              (i + 1, result))
                        continue

                    white_elo = tags.get('WhiteElo')
                    white_elo = int(
                        white_elo
                    ) if white_elo and white_elo.isdigit() else None

                    black_elo = tags.get('BlackElo')
                    black_elo = int(
                        black_elo
                    ) if black_elo and black_elo.isdigit() else None

                    time_control = tags.get("TimeControl")

                    eco = tags.get("ECO")
                    eco = eco[:3] if eco else None

                    fen = tags.get("FEN")

                    board_tag = tags.get("Board")

                    annotator_id = get_id(tags.get("Annotator"), annotator,
                                          ANNOTATOR)

                    source_id = get_id(unicode(orig_filename),
                                       source,
                                       SOURCE,
                                       info=info)

                    self.next_id[GAME] += 1

                    ply_count = tags.get("PlyCount")

                    self.game_data.append({
                        'offset': base_offset + int(offs),
                        'offset8': (int(offs) >> 3) << 3,
                        'event_id': event_id,
                        'site_id': site_id,
                        'date_year': game_year,
                        'date_month': game_month,
                        'date_day': game_day,
                        'round': game_round,
                        'white_id': white_id,
                        'black_id': black_id,
                        'result': result,
                        'white_elo': white_elo,
                        'black_elo': black_elo,
                        'ply_count': ply_count,
                        'eco': eco,
                        'fen': fen,
                        'variant': variant,
                        'board': board_tag,
                        'time_control': time_control,
                        'annotator_id': annotator_id,
                        'source_id': source_id,
                    })

                    i += 1

                    if len(self.game_data) >= self.CHUNK:
                        if self.event_data:
                            self.conn.execute(self.ins_event, self.event_data)
                            self.event_data = []

                        if self.site_data:
                            self.conn.execute(self.ins_site, self.site_data)
                            self.site_data = []

                        if self.player_data:
                            self.conn.execute(self.ins_player,
                                              self.player_data)
                            self.player_data = []

                        if self.annotator_data:
                            self.conn.execute(self.ins_annotator,
                                              self.annotator_data)
                            self.annotator_data = []

                        if self.source_data:
                            self.conn.execute(self.ins_source,
                                              self.source_data)
                            self.source_data = []

                        self.conn.execute(self.ins_game, self.game_data)
                        self.game_data = []

                        if progressbar is not None:
                            GLib.idle_add(progressbar.set_fraction,
                                          i / float(all_games))
                            GLib.idle_add(
                                progressbar.set_text,
                                "%s game headers from %s imported" %
                                (i, basename))
                        else:
                            print(pgnfile, i)

                if self.event_data:
                    self.conn.execute(self.ins_event, self.event_data)
                    self.event_data = []

                if self.site_data:
                    self.conn.execute(self.ins_site, self.site_data)
                    self.site_data = []

                if self.player_data:
                    self.conn.execute(self.ins_player, self.player_data)
                    self.player_data = []

                if self.annotator_data:
                    self.conn.execute(self.ins_annotator, self.annotator_data)
                    self.annotator_data = []

                if self.source_data:
                    self.conn.execute(self.ins_source, self.source_data)
                    self.source_data = []

                if self.game_data:
                    self.conn.execute(self.ins_game, self.game_data)
                    self.game_data = []

                if progressbar is not None:
                    GLib.idle_add(progressbar.set_fraction,
                                  i / float(all_games))
                    GLib.idle_add(
                        progressbar.set_text,
                        "%s game headers from %s imported" % (i, basename))
                else:
                    print(pgnfile, i)
                trans.commit()

                if self.append_pgn:
                    # reopen database to write
                    self.db_handle.close()
                    self.db_handle = protosave(self.chessfile.path,
                                               self.append_pgn)

                    print("Append from %s to %s" %
                          (pgnfile, self.chessfile.path))
                    handle.seek(0)
                    for line in handle:
                        self.db_handle.write(line)
                    self.db_handle.close()

                self.chessfile.handle = protoopen(self.chessfile.path)

            except SQLAlchemyError as e:
                trans.rollback()
                print("Importing %s failed! \n%s" % (pgnfile, e))
Example #29
0
    def do_import(self, filename, info=None, progressbar=None):
        self.progressbar = progressbar
        if progressbar is not None:
            self.pulse = True
            self.timeout_id = GObject.timeout_add(50, self.on_timeout, None)

        # collect new names not in they dict yet
        self.event_data = []
        self.site_data = []
        self.player_data = []
        self.annotator_data = []
        self.source_data = []

        # collect new games and commit them in big chunks for speed
        self.game_data = []
        self.bitboard_data = []
        self.tag_game_data = []

        if filename.startswith("http"):
            filename = download_file(filename, progressbar=progressbar)
            if filename is None:
                return
        else:
            if not os.path.isfile(filename):
                print("Can't open %s" % filename)
                return

        if filename.lower().endswith(".zip") and zipfile.is_zipfile(filename):
            zf = zipfile.ZipFile(filename, "r")
            files = [f for f in zf.namelist() if f.lower().endswith(".pgn")]
        else:
            zf = None
            files = [filename]

        for pgnfile in files:
            basename = os.path.basename(pgnfile)
            if progressbar is not None:
                GLib.idle_add(progressbar.set_text,
                              "Reading %s ..." % basename)
            else:
                print("Reading %s ..." % pgnfile)

            if zf is None:
                cf = pgn_load(protoopen(pgnfile))
            else:
                pgn_file = io.TextIOWrapper(zf.open(pgnfile),
                                            encoding=PGN_ENCODING,
                                            newline='')
                cf = pgn_load(pgn_file)

            if progressbar is not None:
                self.pulse = False

            all_games = len(cf.games)
            self.CHUNK = 1000 if all_games > 5000 else 100

            get_tag = cf._getTag
            get_id = self.get_id
            # use transaction to avoid autocommit slowness
            trans = self.conn.begin()
            try:
                for i in range(all_games):
                    # print i+1#, cf.get_player_names(i)

                    if self.cancel:
                        trans.rollback()
                        return

                    fenstr = get_tag(i, "FEN")
                    variant = cf.get_variant(i)

                    # Fixes for some non statndard Chess960 .pgn
                    if fenstr and variant == "Fischerandom":
                        parts = fenstr.split()
                        parts[0] = parts[0].replace(".", "/").replace("0", "")
                        if len(parts) == 1:
                            parts.append("w")
                            parts.append("-")
                            parts.append("-")
                        fenstr = " ".join(parts)

                    if variant:
                        if variant not in name2variant:
                            print("Unknown variant: %s" % variant)
                            continue
                        variant = name2variant[variant].variant
                        board = LBoard(variant)
                    else:
                        variant = 0
                        board = START_BOARD.clone()

                    if fenstr:
                        try:
                            board.applyFen(fenstr)
                        except SyntaxError as e:
                            print(
                                _("The game #%s can't be loaded, because of an error parsing FEN"
                                  ) % (i + 1), e.args[0])
                            continue
                    elif variant:
                        board.applyFen(FEN_START)

                    movetext = cf.get_movetext(i)

                    movelist = array("H")
                    comments = []
                    cf.error = None

                    # First we try to use simple_parse_movetext()
                    # assuming most games in .pgn contains only moves
                    # without any comments/variations
                    simple = False
                    if not fenstr and not variant:
                        bitboards = []
                        simple = cf.simple_parse_movetext(
                            movetext, board, movelist, bitboards)

                        if cf.error is not None:
                            print("ERROR in game #%s" % (i + 1),
                                  cf.error.args[0])
                            continue

                    # If simple_parse_movetext() find any comments/variations
                    # we restart parsing with full featured parse_movetext()
                    if not simple:
                        movelist = array("H")
                        bitboards = None

                        # in case simple_parse_movetext failed we have to reset our lboard
                        if not fenstr and not variant:
                            board = START_BOARD.clone()

                        # parse movetext to create boards tree structure
                        boards = [board]
                        boards = cf.parse_movetext(movetext,
                                                   boards[0],
                                                   -1,
                                                   pgn_import=True)

                        if cf.error is not None:
                            print("ERROR in game #%s" % (i + 1),
                                  cf.error.args[0])
                            continue

                        # create movelist and comments from boards tree
                        walk(boards[0], movelist, comments)

                    white = get_tag(i, 'White')
                    black = get_tag(i, 'Black')

                    if not movelist:
                        if (not comments) and (not white) and (not black):
                            print("empty game")
                            continue

                    event_id = get_id(get_tag(i, 'Event'), event, EVENT)

                    site_id = get_id(get_tag(i, 'Site'), site, SITE)

                    game_date = get_tag(i, 'Date').strip()
                    try:
                        if game_date and '?' not in game_date:
                            ymd = game_date.split('.')
                            if len(ymd) == 3:
                                game_year, game_month, game_day = map(int, ymd)
                            else:
                                game_year, game_month, game_day = int(
                                    game_date[:4]), None, None
                        elif game_date and '?' not in game_date[:4]:
                            game_year, game_month, game_day = int(
                                game_date[:4]), None, None
                    except:
                        game_year, game_month, game_day = None, None, None

                    game_round = get_tag(i, 'Round')

                    white_fide_id = get_tag(i, 'WhiteFideId')
                    black_fide_id = get_tag(i, 'BlackFideId')

                    white_id = get_id(unicode(white),
                                      player,
                                      PLAYER,
                                      fide_id=white_fide_id)
                    black_id = get_id(unicode(black),
                                      player,
                                      PLAYER,
                                      fide_id=black_fide_id)

                    result = cf.get_result(i)

                    white_elo = get_tag(i, 'WhiteElo')
                    white_elo = int(
                        white_elo
                    ) if white_elo and white_elo.isdigit() else None

                    black_elo = get_tag(i, 'BlackElo')
                    black_elo = int(
                        black_elo
                    ) if black_elo and black_elo.isdigit() else None

                    ply_count = get_tag(i, "PlyCount")

                    time_control = get_tag(i, "TimeControl")

                    eco = get_tag(i, "ECO")
                    eco = eco[:3] if eco else None

                    fen = get_tag(i, "FEN")

                    board_tag = get_tag(i, "Board")

                    annotator_id = get_id(get_tag(i, "Annotator"), annotator,
                                          ANNOTATOR)

                    source_id = get_id(unicode(pgnfile),
                                       source,
                                       SOURCE,
                                       info=info)

                    game_id = self.next_id[GAME]
                    self.next_id[GAME] += 1

                    # annotated game
                    if bitboards is None:
                        for ply, board in enumerate(boards):
                            bb = board.friends[0] | board.friends[1]
                            # Avoid to include mate in x .pgn collections and similar in opening tree
                            if fen and "/pppppppp/8/8/8/8/PPPPPPPP/" not in fen:
                                ply = -1
                            self.bitboard_data.append({
                                'game_id':
                                game_id,
                                'ply':
                                ply,
                                'bitboard':
                                bb - DB_MAXINT_SHIFT,
                            })
                    # simple game
                    else:
                        for ply, bb in enumerate(bitboards):
                            self.bitboard_data.append({
                                'game_id':
                                game_id,
                                'ply':
                                ply,
                                'bitboard':
                                bb - DB_MAXINT_SHIFT,
                            })

                    self.game_data.append({
                        'event_id':
                        event_id,
                        'site_id':
                        site_id,
                        'date_year':
                        game_year,
                        'date_month':
                        game_month,
                        'date_day':
                        game_day,
                        'round':
                        game_round,
                        'white_id':
                        white_id,
                        'black_id':
                        black_id,
                        'result':
                        result,
                        'white_elo':
                        white_elo,
                        'black_elo':
                        black_elo,
                        'ply_count':
                        ply_count,
                        'eco':
                        eco,
                        'fen':
                        fen,
                        'variant':
                        variant,
                        'board':
                        board_tag,
                        'time_control':
                        time_control,
                        'annotator_id':
                        annotator_id,
                        'source_id':
                        source_id,
                        'movelist':
                        movelist.tostring(),
                        'comments':
                        unicode("|".join(comments)),
                    })

                    if len(self.game_data) >= self.CHUNK:
                        if self.event_data:
                            self.conn.execute(self.ins_event, self.event_data)
                            self.event_data = []

                        if self.site_data:
                            self.conn.execute(self.ins_site, self.site_data)
                            self.site_data = []

                        if self.player_data:
                            self.conn.execute(self.ins_player,
                                              self.player_data)
                            self.player_data = []

                        if self.annotator_data:
                            self.conn.execute(self.ins_annotator,
                                              self.annotator_data)
                            self.annotator_data = []

                        if self.source_data:
                            self.conn.execute(self.ins_source,
                                              self.source_data)
                            self.source_data = []

                        self.conn.execute(self.ins_game, self.game_data)
                        self.game_data = []

                        if self.bitboard_data:
                            self.conn.execute(self.ins_bitboard,
                                              self.bitboard_data)
                            self.bitboard_data = []

                        if progressbar is not None:
                            GLib.idle_add(progressbar.set_fraction,
                                          (i + 1) / float(all_games))
                            GLib.idle_add(
                                progressbar.set_text,
                                "%s / %s from %s imported" %
                                (i + 1, all_games, basename))
                        else:
                            print(pgnfile, i + 1)

                if self.event_data:
                    self.conn.execute(self.ins_event, self.event_data)
                    self.event_data = []

                if self.site_data:
                    self.conn.execute(self.ins_site, self.site_data)
                    self.site_data = []

                if self.player_data:
                    self.conn.execute(self.ins_player, self.player_data)
                    self.player_data = []

                if self.annotator_data:
                    self.conn.execute(self.ins_annotator, self.annotator_data)
                    self.annotator_data = []

                if self.source_data:
                    self.conn.execute(self.ins_source, self.source_data)
                    self.source_data = []

                if self.game_data:
                    self.conn.execute(self.ins_game, self.game_data)
                    self.game_data = []

                if self.bitboard_data:
                    self.conn.execute(self.ins_bitboard, self.bitboard_data)
                    self.bitboard_data = []

                if progressbar is not None:
                    GLib.idle_add(progressbar.set_fraction,
                                  (i + 1) / float(all_games))
                    GLib.idle_add(
                        progressbar.set_text, "%s / %s from %s imported" %
                        (i + 1, all_games, basename))
                else:
                    print(pgnfile, i + 1)
                trans.commit()

            except SQLAlchemyError as e:
                trans.rollback()
                print("Importing %s failed! \n%s" % (file, e))