def sgf_properties(self,
save_comments_player=None,
save_comments_class=None,
eval_thresholds=None,
save_analysis=False):
properties = copy.copy(super().sgf_properties())
note = self.note.strip()
if save_analysis and self.analysis_complete:
try:
properties["KT"] = analysis_dumps(self.analysis)
except Exception as e:
print(f"Error in saving analysis: {e}")
if self.points_lost and save_comments_class is not None and eval_thresholds is not None:
show_class = save_comments_class[evaluation_class(
self.points_lost, eval_thresholds)]
else:
show_class = False
comments = properties.get("C", [])
if (self.parent and self.parent.analysis_exists
and self.analysis_exists and
(note or ((save_comments_player or {}).get(self.player, False)
and show_class))):
candidate_moves = self.parent.candidate_moves
top_x = Move.from_gtp(candidate_moves[0]["move"]).sgf(
self.board_size)
best_sq = [
Move.from_gtp(d["move"]).sgf(self.board_size)
for d in candidate_moves[1:] if d["pointsLost"] <= 0.5
]
if best_sq and "SQ" not in properties:
properties["SQ"] = best_sq
if top_x and "MA" not in properties:
properties["MA"] = [top_x]
comments.append(
self.comment(sgf=True, interactive=False) +
SGF_INTERNAL_COMMENTS_MARKER)
if self.is_root:
comments = [
i18n._("SGF start message") + SGF_INTERNAL_COMMENTS_MARKER +
"\n",
*comments,
f"\nSGF generated by {PROGRAM_NAME} {VERSION}{SGF_INTERNAL_COMMENTS_MARKER}\n",
]
properties["CA"] = ["UTF-8"]
properties["AP"] = [f"{PROGRAM_NAME}:{VERSION}"]
if self.shortcut_from:
properties["KTSF"] = [id(self.shortcut_from)]
elif "KTSF" in properties:
del properties["KTSF"]
if self.shortcuts_to:
properties["KTSID"] = [id(self)]
elif "KTSID" in properties:
del properties["KTSID"]
if note:
comments.append(f"{self.note}")
if comments:
properties["C"] = [SGF_SEPARATOR_MARKER.join(comments)]
elif "C" in properties:
del properties["C"]
return properties
def analyze_and_play_policy(node):
nonlocal count, cn
cand = node.candidate_moves
if self.katrain.game is not self:
return # a new game happened
if cand:
move = Move.from_gtp(cand[0]["move"], player=node.next_player)
else:
polmoves = node.policy_ranking
move = polmoves[0][1] if polmoves else Move(None)
if move.is_pass:
if self.current_node == cn:
self.set_current_node(node)
return
count += 1
new_node = GameNode(parent=node, move=move)
if node != cn:
node.remove_shortcut()
cn.add_shortcut(new_node)
self.katrain.controls.move_tree.redraw_tree_trigger()
def set_analysis(result, _partial):
new_node.set_analysis(result)
analyze_and_play_policy(new_node)
self.engines[node.next_player].request_analysis(
new_node,
callback=set_analysis,
priority=-1000,
analyze_fast=True)
def sgf_properties(self,
save_comments_player=None,
save_comments_class=None,
eval_thresholds=None):
properties = copy.copy(super().sgf_properties())
note = self.note.strip()
if self.points_lost and save_comments_class is not None and eval_thresholds is not None:
show_class = save_comments_class[evaluation_class(
self.points_lost, eval_thresholds)]
else:
show_class = False
if (self.parent and self.parent.analysis_ready and self.analysis_ready
and (note or
((save_comments_player or {}).get(self.player, False)
and show_class))):
candidate_moves = self.parent.candidate_moves
top_x = Move.from_gtp(candidate_moves[0]["move"]).sgf(
self.board_size)
best_sq = [
Move.from_gtp(d["move"]).sgf(self.board_size)
for d in candidate_moves[1:] if d["pointsLost"] <= 0.5
]
if best_sq and "SQ" not in properties:
properties["SQ"] = best_sq
if top_x and "MA" not in properties:
properties["MA"] = [top_x]
comment = self.comment(sgf=True, interactive=False)
if comment:
properties["C"] = [
"\n".join(properties.get("C", "")) + comment
]
if self.is_root:
properties["C"] = [
i18n._("SGF start message") + "\n" +
"\n".join(properties.get("C", "")) +
"\nSGF with review generated by KaTrain."
]
if note:
properties["C"] = [
"\n".join(properties.get("C", "")) + f"\nNote: {self.note}"
]
return properties
def analyze_extra(self, mode):
stones = {s.coords for s in self.stones}
cn = self.current_node
engine = self.engines[cn.next_player]
if mode == "extra":
visits = cn.analysis_visits_requested + engine.config["max_visits"]
self.katrain.controls.set_status(i18n._("extra analysis").format(visits=visits))
cn.analyze(engine, visits=visits, priority=-1_000, time_limit=False)
return
elif mode == "sweep":
board_size_x, board_size_y = self.board_size
if cn.analysis_ready:
policy_grid = (
var_to_grid(self.current_node.policy, size=(board_size_x, board_size_y))
if self.current_node.policy
else None
)
analyze_moves = sorted(
[
Move(coords=(x, y), player=cn.next_player)
for x in range(board_size_x)
for y in range(board_size_y)
if (policy_grid is None and (x, y) not in stones) or policy_grid[y][x] >= 0
],
key=lambda mv: -policy_grid[mv.coords[1]][mv.coords[0]],
)
else:
analyze_moves = [
Move(coords=(x, y), player=cn.next_player)
for x in range(board_size_x)
for y in range(board_size_y)
if (x, y) not in stones
]
visits = engine.config["fast_visits"]
self.katrain.controls.set_status(i18n._("sweep analysis").format(visits=visits))
priority = -1_000_000_000
else: # mode=='equalize':
if not cn.analysis_ready:
self.katrain.controls.set_status(i18n._("wait-before-equalize"), self.current_node)
return
analyze_moves = [Move.from_gtp(gtp, player=cn.next_player) for gtp, _ in cn.analysis["moves"].items()]
visits = max(d["visits"] for d in cn.analysis["moves"].values())
self.katrain.controls.set_status(i18n._("equalizing analysis").format(visits=visits))
priority = -1_000
for move in analyze_moves:
cn.analyze(
engine, priority, visits=visits, refine_move=move, time_limit=False
) # explicitly requested so take as long as you need
def analyze_and_play(node):
nonlocal cn, engine_settings
candidates = node.candidate_moves
if self.katrain.game is not self:
return # a new game happened
ai_thoughts = "Move generated by AI self-play\n"
if until_move != "end" and target_b_advantage is not None: # setup pos
if node.depth >= until_move or candidates[0]["move"] == "pass":
self.set_current_node(node)
return
target_score = cn.score + (node.depth - cn.depth + 1) * (
target_b_advantage - cn.score) / (until_move - cn.depth)
max_loss = 5
stddev = min(3, 0.5 + (until_move - node.depth) * 0.15)
ai_thoughts += f"Selecting moves aiming at score {target_score:.1f} +/- {stddev:.2f} with < {max_loss} points lost\n"
if abs(node.score - target_score) < 3 * stddev:
weighted_cands = [
(
move,
math.exp(-0.5 *
(abs(move["scoreLead"] - target_score) /
stddev)**2) *
math.exp(
-0.5 *
(min(0, move["pointsLost"]) / max_loss)**2),
) for i, move in enumerate(candidates)
if move["pointsLost"] < max_loss or i == 0
]
move_info = weighted_selection_without_replacement(
weighted_cands, 1)[0][0]
for move, wt in weighted_cands:
self.katrain.log(
f"{'* ' if move_info == move else ' '} {move['move']} {move['scoreLead']} {wt}",
OUTPUT_EXTRA_DEBUG,
)
ai_thoughts += f"Move option: {move['move']} score {move['scoreLead']:.2f} loss {move['pointsLost']:.2f} weight {wt:.3e}\n"
else: # we're a bit lost, far away from target, just push it closer
move_info = min(
candidates,
key=lambda move: abs(move["scoreLead"] - target_score))
self.katrain.log(
f"* Played {move_info['move']} {move_info['scoreLead']} because score deviation between current score {node.score} and target score {target_score} > {3*stddev}",
OUTPUT_EXTRA_DEBUG,
)
ai_thoughts += f"Move played to close difference between score {node.score:.1f} and target {target_score:.1f} quickly."
self.katrain.log(
f"Self-play until {until_move} target {target_b_advantage}: {len(candidates)} candidates -> move {move_info['move']} score {move_info['scoreLead']} point loss {move_info['pointsLost']}",
OUTPUT_DEBUG,
)
move = Move.from_gtp(move_info["move"],
player=node.next_player)
elif candidates: # just selfplay to end
move = Move.from_gtp(candidates[0]["move"],
player=node.next_player)
else: # 1 visit etc
polmoves = node.policy_ranking
move = polmoves[0][1] if polmoves else Move(None)
if move.is_pass:
if self.current_node == cn:
self.set_current_node(node)
return
new_node = GameNode(parent=node, move=move)
new_node.ai_thoughts = ai_thoughts
if until_move != "end" and target_b_advantage is not None:
self.set_current_node(new_node)
self.katrain.controls.set_status(
i18n._("setup game status message").format(
move=new_node.depth, until_move=until_move),
STATUS_INFO,
)
else:
if node != cn:
node.remove_shortcut()
cn.add_shortcut(new_node)
self.katrain.controls.move_tree.redraw_tree_trigger()
request_analysis_for_node(new_node)
def analyze_extra(self, mode, **kwargs):
stones = {s.coords for s in self.stones}
cn = self.current_node
if mode == "stop":
for e in set(self.engines.values()):
e.terminate_queries()
self.katrain.idle_analysis = False
return
engine = self.engines[cn.next_player]
Clock.schedule_once(self.katrain.analysis_controls.hints.activate, 0)
if mode == "extra":
if kwargs.get("continuous", False):
visits = min(
1_000_000_000,
max(engine.config["max_visits"],
math.ceil(cn.analysis_visits_requested * 1.25)))
else:
visits = cn.analysis_visits_requested + engine.config[
"max_visits"]
self.katrain.controls.set_status(
i18n._("extra analysis").format(visits=visits),
STATUS_ANALYSIS)
self.katrain.controls.set_status(
i18n._("extra analysis").format(visits=visits),
STATUS_ANALYSIS)
cn.analyze(engine,
visits=visits,
priority=-1_000,
region_of_interest=self.region_of_interest,
time_limit=False)
return
if mode == "game":
nodes = self.root.nodes_in_tree
if "visits" in kwargs:
visits = kwargs["visits"]
else:
min_visits = min(node.analysis_visits_requested
for node in nodes)
visits = min_visits + engine.config["max_visits"]
for node in nodes:
node.analyze(engine,
visits=visits,
priority=-1_000_000,
time_limit=False,
report_every=None)
self.katrain.controls.set_status(
i18n._("game re-analysis").format(visits=visits),
STATUS_ANALYSIS)
return
elif mode == "sweep":
board_size_x, board_size_y = self.board_size
if cn.analysis_exists:
policy_grid = (var_to_grid(self.current_node.policy,
size=(board_size_x, board_size_y))
if self.current_node.policy else None)
analyze_moves = sorted(
[
Move(coords=(x, y), player=cn.next_player)
for x in range(board_size_x)
for y in range(board_size_y)
if (policy_grid is None and
(x, y) not in stones) or policy_grid[y][x] >= 0
],
key=lambda mv: -policy_grid[mv.coords[1]][mv.coords[0]],
)
else:
analyze_moves = [
Move(coords=(x, y), player=cn.next_player)
for x in range(board_size_x) for y in range(board_size_y)
if (x, y) not in stones
]
visits = engine.config["fast_visits"]
self.katrain.controls.set_status(
i18n._("sweep analysis").format(visits=visits),
STATUS_ANALYSIS)
priority = -1_000_000_000
elif mode in ["equalize", "alternative", "local"]:
if not cn.analysis_complete and mode != "local":
self.katrain.controls.set_status(
i18n._("wait-before-extra-analysis"), STATUS_INFO,
self.current_node)
return
if mode == "alternative": # also do a quick update on current candidates so it doesn't look too weird
self.katrain.controls.set_status(
i18n._("alternative analysis"), STATUS_ANALYSIS)
cn.analyze(engine,
priority=-500,
time_limit=False,
find_alternatives="alternative")
visits = engine.config["fast_visits"]
else: # equalize
visits = max(d["visits"]
for d in cn.analysis["moves"].values())
self.katrain.controls.set_status(
i18n._("equalizing analysis").format(visits=visits),
STATUS_ANALYSIS)
priority = -1_000
analyze_moves = [
Move.from_gtp(gtp, player=cn.next_player)
for gtp, _ in cn.analysis["moves"].items()
]
else:
raise ValueError("Invalid analysis mode")
for move in analyze_moves:
if cn.analysis["moves"].get(move.gtp(),
{"visits": 0})["visits"] < visits:
cn.analyze(
engine,
priority=priority,
visits=visits,
refine_move=move,
time_limit=False
) # explicitly requested so take as long as you need
def _read_stdout_thread(self):
while self.katago_process is not None:
try:
line = self.katago_process.stdout.readline()
if line:
line = line.decode(errors="ignore").strip()
if line.startswith("{"):
try:
analysis = json.loads(line)
if "gameId" in analysis:
game_id = analysis["gameId"]
if game_id in self.finished_games:
continue
current_game = self.active_games.get(game_id)
new_game = current_game is None
if new_game:
board_size = [analysis["boardXSize"], analysis["boardYSize"]]
placements = {
f"A{bw}": [
Move.from_gtp(move, pl).sgf(board_size)
for pl, move in analysis["initialStones"]
if pl == bw
]
for bw in "BW"
}
game_properties = {k: v for k, v in placements.items() if v}
game_properties["SZ"] = f"{board_size[0]}:{board_size[1]}"
game_properties["KM"] = analysis["rules"]["komi"]
game_properties["RU"] = json.dumps(analysis["rules"])
game_properties["PB"] = analysis["blackPlayer"]
game_properties["PW"] = analysis["whitePlayer"]
current_game = BaseGame(self.katrain, game_properties=game_properties)
self.active_games[game_id] = current_game
last_node = current_game.sync_branch(
[Move.from_gtp(coord, pl) for pl, coord in analysis["moves"]]
)
last_node.set_analysis(analysis)
if new_game:
current_game.set_current_node(last_node)
self.start_time = self.start_time or time.time() - 1
self.move_count += 1
self.visits_count += analysis["rootInfo"]["visits"]
last_move = self.last_move_for_game[game_id]
self.last_move_for_game[game_id] = time.time()
dt = self.last_move_for_game[game_id] - last_move if last_move else 0
self.katrain.log(
f"[{time.time()-self.start_time:.1f}] Game {game_id} Move {analysis['turnNumber']}: {' '.join(analysis['move'])} Visits {analysis['rootInfo']['visits']} Time {dt:.1f}s\t Moves/min {60*self.move_count/(time.time()-self.start_time):.1f} Visits/s {self.visits_count/(time.time()-self.start_time):.1f}",
OUTPUT_DEBUG,
)
self.katrain("update-state")
except Exception as e:
traceback.print_exc()
self.katrain.log(f"Exception {e} in parsing or processing JSON: {line}", OUTPUT_ERROR)
elif "uploaded sgf" in line:
self.uploaded_games_count += 1
else:
self.katrain.log(line, OUTPUT_KATAGO_STDERR)
elif self.katago_process:
self.check_alive(exception_if_dead=False) # stderr will do this
except Exception as e:
self.katrain.log(f"Exception in reading stdout {e}", OUTPUT_DEBUG)
return
def analyze_extra(self, mode, **kwargs):
stones = {s.coords for s in self.stones}
cn = self.current_node
engine = self.engines[cn.next_player]
Clock.schedule_once(self.katrain.analysis_controls.hints.activate, 0)
if mode == "extra":
if kwargs.get("continuous", False):
visits = max(engine.config["max_visits"],
math.ceil(cn.analysis_visits_requested * 1.25))
else:
visits = cn.analysis_visits_requested + engine.config[
"max_visits"]
self.katrain.controls.set_status(
i18n._("extra analysis").format(visits=visits),
STATUS_ANALYSIS)
cn.analyze(engine,
visits=visits,
priority=-1_000,
time_limit=False)
return
if mode == "game":
nodes = self.root.nodes_in_tree
if "visits" in kwargs:
visits = kwargs["visits"]
else:
min_visits = min(node.analysis_visits_requested
for node in nodes)
visits = min_visits + engine.config["max_visits"]
for node in nodes:
node.analyze(engine,
visits=visits,
priority=-1_000_000,
time_limit=False)
self.katrain.controls.set_status(
i18n._("game re-analysis").format(visits=visits),
STATUS_ANALYSIS)
return
elif mode == "sweep":
board_size_x, board_size_y = self.board_size
if cn.analysis_ready:
policy_grid = (var_to_grid(self.current_node.policy,
size=(board_size_x, board_size_y))
if self.current_node.policy else None)
analyze_moves = sorted(
[
Move(coords=(x, y), player=cn.next_player)
for x in range(board_size_x)
for y in range(board_size_y)
if (policy_grid is None and
(x, y) not in stones) or policy_grid[y][x] >= 0
],
key=lambda mv: -policy_grid[mv.coords[1]][mv.coords[0]],
)
else:
analyze_moves = [
Move(coords=(x, y), player=cn.next_player)
for x in range(board_size_x) for y in range(board_size_y)
if (x, y) not in stones
]
visits = engine.config["fast_visits"]
self.katrain.controls.set_status(
i18n._("sweep analysis").format(visits=visits),
STATUS_ANALYSIS)
priority = -1_000_000_000
elif mode == "pass":
board_size_x, board_size_y = self.board_size
analyze_moves = [Move(coords=None, player=cn.next_player)]
visits = 4 * engine.config["fast_visits"]
self.katrain.controls.set_status(
i18n._("pass analysis").format(visits=visits), STATUS_ANALYSIS)
priority = -1_000_000_000
elif mode == "equalize":
if not cn.analysis_ready:
self.katrain.controls.set_status(
i18n._("wait-before-equalize"), STATUS_INFO,
self.current_node)
return
analyze_moves = [
Move.from_gtp(gtp, player=cn.next_player)
for gtp, _ in cn.analysis["moves"].items()
]
visits = max(d["visits"] for d in cn.analysis["moves"].values())
self.katrain.controls.set_status(
i18n._("equalizing analysis").format(visits=visits),
STATUS_ANALYSIS)
priority = -1_000
else:
raise ValueError("Invalid analysis mode")
for move in analyze_moves:
cn.analyze(engine,
priority,
visits=visits,
refine_move=move,
time_limit=False
) # explicitly requested so take as long as you need
Move((random.randint(0, bx - 1), random.randint(0, by - 1)),
player="B").sgf(board_size=game.board_size))
game.root.set_property("AB", list(handicaps))
game._calculate_groups()
gtp = [Move.from_sgf(m, game.board_size, "B").gtp() for m in handicaps]
logger.log(f"Chose handicap placements as {gtp}", OUTPUT_ERROR)
print(f"= {' '.join(gtp)}\n")
sys.stdout.flush()
game.analyze_all_nodes() # re-evaluate root
while engine.queries: # and make sure this gets processed
time.sleep(0.001)
continue
elif line.startswith("set_free_handicap"):
_, *stones = line.split(" ")
game.root.set_property("AB", [
Move.from_gtp(move.upper()).sgf(game.board_size) for move in stones
])
game._calculate_groups()
game.analyze_all_nodes() # re-evaluate root
while engine.queries: # and make sure this gets processed
time.sleep(0.001)
logger.log(
f"Set handicap placements to {game.root.get_list_property('AB')}",
OUTPUT_ERROR)
elif line.startswith("genmove"):
_, player = line.strip().split(" ")
if player[0].upper() != game.current_node.next_player:
logger.log(
f"ERROR generating move: UNEXPECTED PLAYER {player} != {game.current_node.next_player}.",
OUTPUT_ERROR)
print(f"= ??\n")
