def print_beneficial_deviation_analysis(last_meta_game, meta_game, last_meta_prob, verbose=False):
"""
Function to check whether players have found policy of beneficial deviation in current meta_game compared to the last_meta_game
Args:
last_meta_game: List of list of meta_game (One array per game player). The meta game to compare against
meta_game: List of list of meta_game (One array per game player). Current iteration's meta game. Same length with last_meta_game, and each element in meta_game has to include all entries in last_meta_game's corresponding elements
last_meta_prob: nash equilibrium of last g_psro_iteration. List of list. Last iteration
Returns:
a list of length num_players, indicating the number of beneficial deviations for each player from last_meta_prob
"""
num_player = len(last_meta_prob)
num_new_pol = [ meta_game[0].shape[i]-len(last_meta_prob[i]) for i in range(num_player)]
num_pol = [ meta_game[0].shape[i] for i in range(num_player)]
prob_matrix = meta_strategies.general_get_joint_strategy_from_marginals(last_meta_prob)
this_meta_prob = [np.append(last_meta_prob[i],[0 for _ in range(num_new_pol[i])]) for i in range(num_player)]
beneficial_deviation = [0 for _ in range(num_player)]
for i in range(num_player):
ne_payoff = np.sum(last_meta_game[i]*prob_matrix)
# iterate through player's new policy
for j in range(num_new_pol[i]):
dev_prob = this_meta_prob.copy()
dev_prob[i] = np.zeros(num_pol[i])
dev_prob[i][len(last_meta_prob[i])+j] = 1
new_prob_matrix = meta_strategies.general_get_joint_strategy_from_marginals(dev_prob)
dev_payoff = np.sum(meta_game[i]*new_prob_matrix)
if ne_payoff < dev_payoff:
beneficial_deviation[i] += 1
if verbose:
print("\n---------------------------\nBeneficial Deviation :")
for p in range(len(beneficial_deviation)):
print('player '+str(p)+':',beneficial_deviation[p])
return beneficial_deviation
def strategy_regret(meta_games, subgame_index, ne=None, subgame_ne=None):
"""
Calculate the strategy regret based on a complete payoff matrix for PSRO.
strategy_regret of player equals to nash_payoff in meta_game - fix opponent nash strategy, player deviates to subgame_nash
Assume all players have the same number of policies.
:param meta_games: meta_games in PSRO
:param subgame_index: subgame to evaluate, redundant if subgame_nash supplied
:param: nash: equilibrium vector
:param: subgame_ne: equilibrium vector
:return: a list of regret, one for each player.
"""
num_players = len(meta_games)
num_new_pol = np.shape(meta_games[0])[0] - subgame_index
ne = nash_solver(meta_games, solver="gambit") if not ne else ne
index = [list(np.arange(subgame_index)) for _ in range(num_players)]
submeta_games = [ele[np.ix_(*index)] for ele in meta_games]
subgame_ne = nash_solver(submeta_games, solver="gambit") if not subgame_ne else subgame_ne
nash_prob_matrix = meta_strategies.general_get_joint_strategy_from_marginals(ne)
regrets = []
for i in range(num_players):
ne_payoff = np.sum(meta_games[i]*nash_prob_matrix)
dev_prob = ne.copy()
dev_prob[i] = list(np.append(subgame_ne[i],[0 for _ in range(num_new_pol)]))
dev_prob_matrix = meta_strategies.general_get_joint_strategy_from_marginals(dev_prob)
subgame_payoff = np.sum(meta_games[i]*dev_prob_matrix)
regrets.append(ne_payoff-subgame_payoff)
return regrets
def regret(meta_games, subgame_index, subgame_ne=None, start_index=0):
"""
Calculate the regret based on a complete payoff matrix for PSRO
In subgame, each player could have different number of strategies
:param meta_games: meta_games in PSRO
:param subgame_index: last policy index in subgame.
subgame_index-start_index+1=number of policy
int/list. If int, players have same num of strategies
:param start_index: starting index for the subgame.
int/list. If int, assume subgame in all num_players dimension
have the same index
:param: subgame_ne: subgame nash equilibrium vector.
:return: a list of regret, one for each player.
"""
num_policy = np.array(np.shape(meta_games[0]))
num_players = len(meta_games)
subgame_index = np.ones(num_players,dtype=int)*subgame_index \
if np.isscalar(subgame_index) else subgame_index
start_index = np.ones(num_players,dtype=int)*start_index \
if np.isscalar(start_index) else start_index
if not sum(num_policy != subgame_index-start_index+1):
print("The subgame is same as the full game. Return zero regret.")
return np.zeros(num_players)
num_new_pol_back = num_policy - subgame_index - 1
index = [list(np.arange(start_index[i],subgame_index[i]+1)) for i in range(num_players)]
submeta_games = [ele[np.ix_(*index)] for ele in meta_games]
nash = nash_solver(submeta_games, solver="gambit") if not subgame_ne else subgame_ne
prob_matrix = meta_strategies.general_get_joint_strategy_from_marginals(nash)
this_meta_prob = [np.concatenate(([0 for _ in range(start_index[i])], nash[i], [0 for _ in range(num_new_pol_back[i])])) for i in range(num_players)]
nash_payoffs = []
deviation_payoffs = []
for i in range(num_players):
ne_payoff = np.sum(submeta_games[i]*prob_matrix)
# iterate through player's new policy
dev_payoff = []
for j in range(start_index[i] + num_new_pol_back[i]):
dev_prob = this_meta_prob.copy()
dev_prob[i] = np.zeros(num_policy[i])
if j < start_index[i]:
dev_prob[i][j] = 1
else:
dev_prob[i][subgame_index[i]+j-start_index[i]+1] = 1
new_prob_matrix = meta_strategies.general_get_joint_strategy_from_marginals(dev_prob)
dev_payoff.append(np.sum(meta_games[i]*new_prob_matrix))
deviation_payoffs.append(dev_payoff-ne_payoff)
nash_payoffs.append(ne_payoff)
regret = [np.max(ele) for ele in deviation_payoffs]
return regret
def regret(meta_games, subgame_index, subgame_ne=None):
"""
Calculate the regret based on a complete payoff matrix for PSRO
Assume all players have the same number of policies
:param meta_games: meta_games in PSRO
:param subgame_index: the subgame to evaluate. Redundant when subgame_ne is supplied
:param: subgame_ne: subgame nash equilibrium vector.
:return: a list of regret, one for each player.
"""
num_policy = np.shape(meta_games[0])[0]
num_players = len(meta_games)
if num_policy == subgame_index:
print("The subgame is same as the full game. Return zero regret.")
return np.zeros(num_players)
num_new_pol = num_policy - subgame_index
index = [list(np.arange(subgame_index)) for _ in range(num_players)]
submeta_games = [ele[np.ix_(*index)] for ele in meta_games]
nash = nash_solver(submeta_games, solver="gambit") if not subgame_ne else subgame_ne
prob_matrix = meta_strategies.general_get_joint_strategy_from_marginals(nash)
this_meta_prob = [np.append(nash[i],[0 for _ in range(num_new_pol)]) for i in range(num_players)]
nash_payoffs = []
deviation_payoffs = []
for i in range(num_players):
ne_payoff = np.sum(submeta_games[i]*prob_matrix)
# iterate through player's new policy
dev_payoff = []
for j in range(num_new_pol):
dev_prob = this_meta_prob.copy()
dev_prob[i] = np.zeros(num_policy)
dev_prob[i][subgame_index+j] = 1
new_prob_matrix = meta_strategies.general_get_joint_strategy_from_marginals(dev_prob)
dev_payoff.append(np.sum(meta_games[i]*new_prob_matrix))
deviation_payoffs.append(dev_payoff-ne_payoff)
nash_payoffs.append(ne_payoff)
regret = np.maximum(np.max(deviation_payoffs,axis=1),0)
return regret
def get_mixed_payoff(self, strategy_list, strategy_support):
"""
Check if the payoff exists for the profile given. If not, return False
Params:
strategy_list : list of list, policy index for each player
strategy_support : list of list, policy support probability for each player
Returns:
payoffs : payoff for each player in the profile
"""
if np.any(np.isnan(self._meta_games[0][np.ix_(*strategy_list)])):
return False
meta_game = [ele[np.ix_(*strategy_list)] for ele in self._meta_games]
prob_matrix = meta_strategies.general_get_joint_strategy_from_marginals(
strategy_support)
payoffs = []
for i in range(self._num_players):
payoffs.append(np.sum(meta_game[i] * prob_matrix))
return payoffs
def rollout(env, strategies, strategy_support, sims_per_entry=1000):
"""
Evaluate player's mixed strategy with support in env.
Params:
env : an open_spiel env
strategies : list of list, each list containing a player's strategy
strategy_support : mixed_strategy support probability vector
sims_per_entry : number of episodes for each pure strategy profile to sample
Return:
a list of players' payoff
"""
num_players = len(strategies)
num_strategies = [len(ele) for ele in strategies]
prob_matrix = meta_strategies.general_get_joint_strategy_from_marginals(strategy_support)
payoff_tensor = np.zeros([num_players]+num_strategies)
for ind in itertools.product(*[np.arange(ele) for ele in num_strategies]):
strat = [strategies[i][ind[i]] for i in range(num_players)]
pure_payoff = sample_episodes(env, strat, sims_per_entry)
payoff_tensor[tuple([...]+list(ind))] = pure_payoff
return [np.sum(payoff_tensor[i]*prob_matrix) for i in range(num_players)]