def test__update_current_policy(self): game = pyspiel.load_game("kuhn_poker") tabular_policy = policy.TabularPolicy(game) cumulative_regrets = np.arange(0, 12 * 2).reshape((12, 2)) expected_policy = cumulative_regrets / np.sum( cumulative_regrets, axis=-1, keepdims=True) nodes_indices = { u"0": 0, u"0pb": 1, u"1": 2, u"1pb": 3, u"2": 4, u"2pb": 5, u"1p": 6, u"1b": 7, u"2p": 8, u"2b": 9, u"0p": 10, u"0b": 11, } # pylint: disable=g-complex-comprehension info_state_nodes = { key: cfr._InfoStateNode( legal_actions=[0, 1], index_in_tabular_policy=None, cumulative_regret=dict(enumerate(cumulative_regrets[index])), cumulative_policy=None) for key, index in nodes_indices.items() } # pylint: enable=g-complex-comprehension cfr._update_current_policy(tabular_policy, info_state_nodes) np.testing.assert_array_equal(expected_policy, tabular_policy.action_probability_array)
def evaluate_and_update_policy(self): """Performs a single step of policy evaluation and policy improvement.""" self._iteration += 1 if self._alternating_updates: for current_player in range(self._game.num_players()): self._compute_counterfactual_regret_for_player( self._root_node, policies=None, reach_probabilities=np.ones(self._game.num_players() + 1), player=current_player) for info_state in self._player_nodes[current_player]: for action in info_state.cumulative_regret.keys(): if info_state.cumulative_regret[action] >= 0: info_state.cumulative_regret[action] *= ( self._iteration**self.alpha / (self._iteration**self.alpha + 1)) else: info_state.cumulative_regret[action] *= ( self._iteration**self.beta / (self._iteration**self.beta + 1)) cfr._update_current_policy(self._current_policy, self._info_state_nodes) # pylint: disable=protected-access