def game_from_sgf(sgfdata, game_number=0): col = SGFParser(sgfdata).parse() cur = col.cursor(game_number) size = int(cur.node['SZ'][0]) game = Game(size) #build moves list -- if it isn't in order in the file, bad things happen add_black = [] add_white = [] moves = [] while 1: if cur.node.has_key('AB'): add_black += map(sgf_to_move, cur.node['AB']) elif cur.node.has_key('AW'): add_white += map(sfg_to_move, cur.node['AW']) elif cur.node.has_key('B'): moves.append(sgf_to_move(cur.node['B'][0])) elif cur.node.has_key('W'): moves.append(sgf_to_move(cur.node['W'][0])) if cur.atEnd: break cur.next() #add black and white (setup positions) if add_black: game.add_black(add_black) if add_white: game.add_white(add_white) #move the game along for move in moves: game.make_move(move) return game
def game_from_sgf(sgfdata, game_number=0): col = SGFParser(sgfdata).parse() cur = col.cursor(game_number) size = int(cur.node['SZ'][0]) game = Game(size) #build moves list -- if it isn't in order in the file, bad things happen add_black = [] add_white = [] moves = [] while 1: if cur.node.has_key('AB'): add_black += map(sgf_to_move, cur.node['AB']) elif cur.node.has_key('AW'): add_white += map(sfg_to_move, cur.node['AW']) elif cur.node.has_key('B'): moves.append(sgf_to_move(cur.node['B'][0])) elif cur.node.has_key('W'): moves.append(sgf_to_move(cur.node['W'][0])) if cur.atEnd: break cur.next() #add black and white (setup positions) if add_black: game.add_black(add_black) if add_white: game.add_white(add_white) #move the game along for move in moves: game.make_move(move) return game
def parse_sgf(path_to_sgf): """Return parsed Collection from sgf""" if not os.path.exists(path_to_sgf): raise FileNotFoundError("No such file: %s" % path_to_sgf) with open(path_to_sgf, 'r', encoding="utf-8") as sgf_file: data = "".join([line for line in sgf_file]) return SGFParser(data).parse()
def process_sgf_file(fin, fout): sgfdata = fin.read() col = SGFParser(sgfdata).parse() for gametree in col: try: process_gametree(gametree, fout) except UnknownNode: # Try next game tree in this file if DEBUG: print >>sys.stderr, "Unknown Node" continue
class BotAnalyzer: def __init__(self, path_to_sgf, bot_config): self._path_to_sgf = path_to_sgf self._bot_config = bot_config self.sgf_data = None self.cursor = None self.analyzer = None self.bot = None self.base_dir = None self.moves_to_analyze = {} self.moves_to_variations = {} self.best_moves = {} self.all_stats = {} self.all_move_lists = {} def factory(self): kwargs = {'board_size': self.board_size, 'komi': self.komi, 'handicap': self.handicap} bot_settings = BOTS[self._bot_config] kwargs.update(bot_settings) if bot_settings['bot_type'] == 'leela': return LeelaCLI(**kwargs) elif bot_settings['bot_type'] == 'leela-zero': return LeelaZeroCLI(**kwargs) @property def board_size(self): node_boardsize = self.cursor.node.get('SZ') if node_boardsize: board_size = int(node_boardsize.data[0]) if board_size != 19: logger.warning("Board size is not 19 so analysis could be very inaccurate.") else: board_size = 19 return board_size @property def handicap(self): node_handicap = self.cursor.node.get('HA') if node_handicap: return int(node_handicap.data[0]) else: return 0 @property def japanese_rules(self): node_rules = self.cursor.node.get('RU') return node_rules and node_rules.data[0].lower() in ['jp', 'japanese', 'japan'] @property def komi(self): """ Returns adjusted komi.""" node_komi = self.cursor.node.get('KM') if node_komi: komi = round(float(node_komi.data[0]), 1) if self.japanese_rules: komi += self.handicap elif self.handicap: komi = 0.5 else: komi = 6.5 if self.japanese_rules else 7.5 return komi def parse_sgf_file(self): """ Returns parsed Collection from sgf""" with open(self._path_to_sgf, 'r', encoding="utf-8") as sgf_file: data = "".join([line for line in sgf_file]) self.sgf_data = SGFParser(data).parse() def save_to_file(self): file_name, file_ext = os.path.splitext(self._path_to_sgf) path_to_save = f"{file_name}_{self._bot_config}{file_ext}" with open(path_to_save, mode='w', encoding='utf-8') as f: f.write(str(self.sgf_data)) def graph_winrates(self): import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt if len(self.all_stats) <= 2: return first_move_num = min(self.all_stats.keys()) last_move_num = max(self.all_stats.keys()) x = [] y = [] for move_num in sorted(self.all_stats.keys()): if 'winrate' not in self.all_stats[move_num]: continue x.append(move_num) y.append(self.all_stats[move_num]['winrate']) plt.figure() # fill graph with horizontal coordinate lines, step 0.25 for xc in np.arange(0, 1, 0.025): plt.axhline(xc, first_move_num, last_move_num, linewidth=0.04, color='0.7') # add single central horizontal line plt.axhline(0.50, first_move_num, last_move_num, linewidth=0.3, color='0.2') # main graph of win rate changes plt.plot(x, y, color='#ff0000', marker='.', markersize=2.5, linewidth=0.6) # set range limits for x and y axes plt.xlim(0, last_move_num) plt.ylim(0, 1) # set size of numbers on axes plt.yticks(np.arange(0, 1.05, 0.05), fontsize=6) plt.yticks(fontsize=6) # add labels to axes plt.xlabel("Move Number", fontsize=10) plt.ylabel("Win Rate", fontsize=12) # in this script for pdf it use the same file name as provided sgf file to avoid extra parameters file_name = os.path.splitext(self._path_to_sgf)[0] file_name = f"{file_name}_{self._bot_config}.pdf" plt.savefig(file_name, dpi=200, format='pdf', bbox_inches='tight') plt.close() def add_moves_to_bot(self): this_move = None if 'W' in self.cursor.node.keys(): this_move = self.cursor.node['W'].data[0] self.bot.add_move_to_history('white', this_move) if 'B' in self.cursor.node.keys(): this_move = self.cursor.node['B'].data[0] self.bot.add_move_to_history('black', this_move) # SGF commands to add black or white stones, often used for setting up handicap and such if 'AB' in self.cursor.node.keys(): for move in self.cursor.node['AB'].data: self.bot.add_move_to_history('black', move) if 'AW' in self.cursor.node.keys(): for move in self.cursor.node['AW'].data: self.bot.add_move_to_history('white', move) return this_move def next_move_pos(self): mv = None if not self.cursor.atEnd: self.cursor.next() if 'W' in self.cursor.node.keys(): mv = self.cursor.node['W'].data[0] if 'B' in self.cursor.node.keys(): mv = self.cursor.node['B'].data[0] self.cursor.previous() return mv def do_analyze(self): ckpt_hash = f"{self.bot.history_hash()}_{self.bot.time_per_move}_sec" ckpt_fn = os.path.join(self.base_dir, ckpt_hash) if os.path.exists(ckpt_fn): logger.debug("Loading checkpoint file: %s", ckpt_fn) with open(ckpt_fn, 'rb') as ckpt_file: stats, move_list = pickle.load(ckpt_file) else: self.bot.clear_board() self.bot.go_to_position() stats, move_list = self.bot.analyze() with open(ckpt_fn, 'wb') as ckpt_file: pickle.dump((stats, move_list), ckpt_file) return stats, move_list def prepare(self): """ Stores moves to analyze and wipes comments if needed""" base_hash = hashlib.md5(str(self.sgf_data).encode()).hexdigest() self.base_dir = os.path.join(settings.CHECKPOINTS_DIR.format(self._bot_config), base_hash) os.makedirs(self.base_dir, exist_ok=True) move_num = -1 while not self.cursor.atEnd: self.cursor.next() move_num += 1 if CONFIG['move_from'] <= move_num + 1 <= CONFIG['move_till']: self.moves_to_analyze[move_num] = True node_comment = self.cursor.node.get('C') if node_comment and CONFIG['wipe_comments']: node_comment.data[0] = "" def analyze_main_line(self): logger.info("Started analyzing main line.") move_num = -1 prev_stats = {} prev_move_list = [] has_prev = False previous_player = None logger.info(f"Executing analysis for %d moves", len(self.moves_to_analyze)) moves_count = 0 self.cursor.reset() self.bot = self.factory() self.bot.time_per_move = CONFIG['analyze_time'] self.bot.start() # analyze main line, without variations while not self.cursor.atEnd: self.cursor.next() move_num += 1 this_move = self.add_moves_to_bot() current_player = 'black' if 'W' in self.cursor.node else 'white' if previous_player == current_player: raise BotException('Two consecutive moves.') if move_num in self.moves_to_analyze: stats, move_list = self.do_analyze() # Here we store ALL statistics self.all_stats[move_num] = stats self.all_move_lists[move_num] = move_list if move_list and 'winrate' in move_list[0]: self.best_moves[move_num] = move_list[0] delta = 0.0 if 'winrate' in stats and (move_num - 1) in self.best_moves: if this_move != self.best_moves[move_num - 1]['pos']: delta = stats['winrate'] - self.best_moves[move_num - 1]['winrate'] delta = min(0.0, (-delta if self.bot.whose_turn() == "black" else delta)) if -delta > CONFIG['analyze_threshold']: (delta_comment, delta_lb_values) = annotations.format_delta_info(delta, this_move, self.board_size) annotations.annotate_sgf(self.cursor, delta_comment, delta_lb_values, []) if has_prev and delta <= -CONFIG['variations_threshold']: self.moves_to_variations[move_num - 1] = True if -delta > CONFIG['analyze_threshold']: logger.warning("Move %d: %s %s is a mistake (winrate dropped by %.2f%%)", move_num + 1, previous_player, convert_position(self.board_size, this_move), -delta * 100) next_game_move = self.next_move_pos() annotations.annotate_sgf(self.cursor, annotations.format_winrate(stats, move_list, self.board_size, next_game_move), [], []) if has_prev and ((move_num - 1) in self.moves_to_analyze and -delta > CONFIG['analyze_threshold'] or ( move_num - 1) in self.moves_to_variations): (analysis_comment, lb_values, tr_values) = annotations.format_analysis( prev_stats, filter_move_list(prev_move_list), this_move, self.board_size) self.cursor.previous() # adding comment to sgf with suggested alternative variations annotations.annotate_sgf(self.cursor, analysis_comment, lb_values, tr_values) self.cursor.next() prev_stats = stats prev_move_list = move_list has_prev = True self.save_to_file() self.graph_winrates() if 'winrate' in stats \ and (1 - CONFIG['stop_on_winrate'] > stats['winrate'] or stats['winrate'] > CONFIG['stop_on_winrate']): break moves_count += 1 logger.info("Analysis done for %d/%d move.", moves_count, len(self.moves_to_analyze)) else: prev_stats = {} prev_move_list = [] has_prev = False previous_player = current_player logger.info("Finished analyzing main line.") def do_variations(self, move_num): stats = self.all_stats[move_num] move_list = filter_move_list(self.all_move_lists[move_num]) game_move = self.next_move_pos() rootcolor = self.bot.whose_turn() leaves = [] tree = {"children": [], "is_root": True, "history": [], "explored": False, "stats": stats, "move_list": move_list, "color": rootcolor} def expand(node, stats, move_list): assert node["color"] in ['white', 'black'] for move in move_list: # Don't expand on the actual game line as a variation! if node["is_root"] and move["pos"] == game_move: continue subhistory = node["history"][:] subhistory.append(move["pos"]) clr = "white" if node["color"] == "black" else "black" child = {"children": [], "is_root": False, "history": subhistory, "explored": False, "stats": {}, "move_list": [], "color": clr} node["children"].append(child) leaves.append(child) node["stats"] = stats node["move_list"] = move_list node["explored"] = True for leaf_idx in range(len(leaves)): if leaves[leaf_idx] is node: del leaves[leaf_idx] break def analyze_and_expand(node): for mv in node["history"]: self.bot.add_move_to_history(self.bot.whose_turn(), mv) stats, move_list = self.do_analyze() expand(node, stats, filter_move_list(move_list)) self.bot.pop_move_from_history(len(node['history'])) self.save_to_file() expand(tree, stats, move_list) for i in range(CONFIG['variations_depth']): if len(leaves) > 0: for leaf in leaves: if not len(leaf['history']) > CONFIG['variations_depth']: analyze_and_expand(leaf) def advance(color, mv): found_child_idx = None clr = 'W' if color == 'white' else 'B' for j in range(len(self.cursor.children)): if clr in self.cursor.children[j].keys() and self.cursor.children[j][clr].data[0] == mv: found_child_idx = j if found_child_idx is not None: self.cursor.next(found_child_idx) else: nnode = Node() nnode.add_property(Property(clr, [mv])) self.cursor.append_node(nnode) self.cursor.next(len(self.cursor.children) - 1) def record(node): if not node["is_root"]: annotations.annotate_sgf(self.cursor, annotations.format_winrate(node["stats"], node["move_list"], self.board_size, None), [], []) move_list_to_display = [] # Only display info for the principal variation or for lines that have been explored. for i in range(len(node["children"])): child = node["children"][i] if child is not None and (i == 0 or child["explored"]): move_list_to_display.append(node["move_list"][i]) (analysis_comment, lb_values, tr_values) = annotations.format_analysis(node["stats"], move_list_to_display, None, self.board_size) annotations.annotate_sgf(self.cursor, analysis_comment, lb_values, tr_values) for i in range(len(node["children"])): child = node["children"][i] if child is not None: if child["explored"]: advance(node["color"], child["history"][-1]) record(child) self.cursor.previous() # Only show variations for the principal line, to prevent info overload elif i == 0: pv = node["move_list"][i]["pv"] color = node["color"] if CONFIG['num_to_show']: num_to_show = min(len(pv), CONFIG['num_to_show']) else: num_to_show = len(pv) for k in range(int(num_to_show)): advance(color, pv[k]) color = 'black' if color == 'white' else 'white' for k in range(int(num_to_show)): self.cursor.previous() record(tree) def analyze_variations(self): logger.info("Started deep analysis of mistakes.") move_num = -1 self.cursor.reset() self.bot.reset() self.bot.time_per_move = CONFIG['variations_time'] self.add_moves_to_bot() logger.info("Exploring variations for %d moves with %d depth.", len(self.moves_to_variations), CONFIG['variations_depth']) moves_count = 0 while not self.cursor.atEnd: self.cursor.next() move_num += 1 self.add_moves_to_bot() if move_num not in self.moves_to_variations: continue stats, move_list = self.all_stats[move_num], self.all_move_lists[move_num] if 'bookmoves' in stats or len(move_list) <= 0: continue self.do_variations(move_num) moves_count += 1 logger.info("Analyzed %d/%d mistakes.", moves_count, len(self.moves_to_variations)) self.save_to_file() logger.info("Finished deep analysis of mistakes.") def run(self): logger.info("Started analyzing file: %s", os.path.basename(self._path_to_sgf)) self.parse_sgf_file() self.cursor = self.sgf_data.cursor() try: self.prepare() self.analyze_main_line() self.analyze_variations() except KeyboardInterrupt: pass except: logger.exception("Exception during analysis.") finally: self.bot.stop() logger.info("Finished analyzing file: %s", os.path.basename(self._path_to_sgf))
def parse_sgf_file(self): """ Returns parsed Collection from sgf""" with open(self._path_to_sgf, 'r', encoding="utf-8") as sgf_file: data = "".join([line for line in sgf_file]) self.sgf_data = SGFParser(data).parse()
def parse_sgf_data(self, data): self.sgf_data = SGFParser(data).parse()
path = sys.argv[1] os.makedirs('cleaned_sgfs', exist_ok=True) if os.path.isdir(path): os.makedirs(os.path.join('cleaned_sgfs', path), exist_ok=True) for s in os.listdir(path): if os.path.splitext(s)[1] == '.sgf': game_list.append(os.path.join(path, s)) else: game_list.append(path) count = 0 for game in game_list: with open(game, 'r') as f: sgf = SGFParser(f.read()).parse().cursor() new_sgf = GameTree() while not sgf.atEnd: new_node = Node() for prop in sgf.node: if sgf.node[prop].label != 'C': new_node.add_property(sgf.node[prop]) new_sgf.append_node(new_node) sgf.next() with open(os.path.join('cleaned_sgfs', game), 'w') as f: f.write(str(new_sgf)) count += 1 if count % 1000: