def get_move_probabilities(self, state, legal_actions):
result = np.zeros(len(legal_actions))
if state.status == 'propose':
# Only consider proposals with yourself
for i, act in enumerate(legal_actions):
result[i] += 1 if self.player in act.proposal else 0
elif state.status == 'vote':
result += 1
if self.player in state.proposal:
# Vote up most proposals with yourself
result[legal_actions.index(VoteAction(up=True))] += 5
elif self.player not in state.proposal:
if state.propose_count == 4 and not self.is_evil:
# Vote up most proposals on the final round if you're good
result[legal_actions.index(VoteAction(up=True))] += 5
else:
# Vote down most proposals which don't contain you.
result[legal_actions.index(VoteAction(up=False))] += 5
elif state.status == 'run':
result += 1
if self.is_evil:
# Fail most missions unless it's the first one
if state.fails + state.succeeds == 0:
result[legal_actions.index(MissionAction(fail=False))] += 5
else:
result[legal_actions.index(MissionAction(fail=True))] += 5
elif state.status == 'merlin':
# Try to pick merlin based off of hidden states
for hidden_state in self.hidden_states:
merlin = hidden_state.index('merlin')
result[legal_actions.index(
PickMerlinAction(merlin=merlin))] += 1
return result / np.sum(result)
def get_opponent_moves_and_probs(state,
hidden_state,
player,
no_tremble=False):
legal_actions = state.legal_actions(player, hidden_state)
if len(legal_actions) == 1:
return legal_actions, np.ones(1)
move_probs = np.zeros(len(legal_actions))
if state.status == 'propose':
for i, act in enumerate(legal_actions):
if player in act.proposal:
move_probs[i] += 1
elif state.status == 'vote':
vote_up = player in state.proposal
move_probs[legal_actions.index(VoteAction(up=vote_up))] = 1
elif state.status == 'run':
# must be evil since we have two options here.
move_probs[legal_actions.index(MissionAction(fail=True))] = 1
elif state.status == 'merlin':
for i, act in enumerate(legal_actions):
if hidden_state[act.merlin] in GOOD_ROLES:
move_probs[i] += 1
move_probs /= np.sum(move_probs)
if no_tremble:
return legal_actions, move_probs
tremble = np.ones(len(legal_actions)) / len(legal_actions)
return legal_actions, (
1 - OPPONENT_TREMBLE) * move_probs + OPPONENT_TREMBLE * tremble
def get_move_probabilities(self, state, legal_actions):
if len(legal_actions) == 1:
return np.array([1.0])
if state.status == 'propose':
probs = np.zeros(len(legal_actions))
propose_strategy = self.node['propose_strat'][self.perspective]
propose_options = self.node['propose_options']
for strategy_prob, proposal_bitstring in zip(propose_strategy, propose_options):
action = ProposeAction(proposal=bitstring_to_proposal(proposal_bitstring))
probs[legal_actions.index(action)] = strategy_prob
# print probs
# print_move_probs(probs, legal_actions)
return probs
elif state.status == 'vote':
probs = np.zeros(len(legal_actions))
vote_strategy = self.node['vote_strat'][self.player][self.perspective]
for strategy_prob, vote_up in zip(vote_strategy, [False, True]):
action = VoteAction(up=vote_up)
probs[legal_actions.index(action)] = strategy_prob
return probs
elif state.status == 'run':
probs = np.zeros(len(legal_actions))
mission_strategy = self.node['mission_strat'][self.player][self.perspective]
for strategy_prob, fail in zip(mission_strategy, [False, True]):
action = MissionAction(fail=fail)
probs[legal_actions.index(action)] = strategy_prob
return probs
elif state.status == 'merlin':
# print np.array(self.node['merlin_strat'][self.player][self.perspective])
return np.array(self.node['merlin_strat'][self.player][self.perspective])
def handle_round(tree_roots, state, hidden_state, round_):
last_proposal = None
for proposal_num in ['1', '2', '3', '4', '5']:
proposal = last_proposal = round_[proposal_num]
assert state.proposer == proposal['proposer'], "idk"
assert state.propose_count == int(proposal_num) - 1, "idk2"
moves = [ProposeAction(proposal=tuple(sorted(proposal['team'])))]
tree_roots, state = deal_with_transition(tree_roots, state, moves,
hidden_state)
assert state.status == 'vote'
moves = [
VoteAction(up=(vote == 'Approve')) for vote in proposal['votes']
]
tree_roots, state = deal_with_transition(tree_roots, state, moves,
hidden_state)
if state.status == 'run':
break
secret_votes = sorted(zip(last_proposal['team'], round_['mission']))
moves = [
MissionAction(fail=(vote == "Fail")) for player, vote in secret_votes
]
tree_roots, state = deal_with_transition(tree_roots, state, moves,
hidden_state)
if state.status == 'merlin':
assert 'findMerlin' in round_, "wat"
find_merlin = round_['findMerlin']
assert hidden_state[find_merlin['assassin']] == 'assassin', "wat"
moves = [
PickMerlinAction(merlin=find_merlin['merlin_guess'])
for _ in hidden_state
]
tree_roots, state = deal_with_transition(tree_roots, state, moves,
hidden_state)
return tree_roots, state
def handle_round(game, state, hidden_state, assassin_bot, assassin_player,
round_):
last_proposal = None
for proposal_num in ['1', '2', '3', '4', '5']:
proposal = last_proposal = round_[proposal_num]
assert state.proposer == proposal['proposer']
assert state.propose_count == int(proposal_num) - 1
moves = [ProposeAction(proposal=tuple(sorted(proposal['team'])))]
state = handle_transition(state, hidden_state, moves, assassin_bot,
assassin_player)
assert state.status == 'vote'
moves = [
VoteAction(up=(vote == 'Approve')) for vote in proposal['votes']
]
state = handle_transition(state, hidden_state, moves, assassin_bot,
assassin_player)
if state.status == 'run':
break
secret_votes = sorted(zip(last_proposal['team'], round_['mission']))
moves = [
MissionAction(fail=(vote == "Fail")) for player, vote in secret_votes
]
state = handle_transition(state, hidden_state, moves, assassin_bot,
assassin_player)
return state
def move_index_to_move(move_index, state):
if state.status == 'merlin':
return PickMerlinAction(merlin=move_index)
elif state.status == 'propose':
size, _ = state.MISSION_SIZES[state.succeeds + state.fails]
mapping = INDEX_TO_PROPOSAL_2 if size == 2 else INDEX_TO_PROPOSAL_3
return ProposeAction(proposal=bitstring_to_proposal(mapping[move_index]))
elif state.status == 'vote':
return VoteAction(up=bool(move_index))
else:
return MissionAction(fail=bool(move_index))
def legal_actions(self, player, hidden_state):
"""
Returns the legal actions of the player from this state, given a hidden state
"""
assert player in self.moving_players(), "Asked a non-moving player legal actions"
if self.status == 'merlin':
return [PickMerlinAction(merlin=p) for p in range(self.NUM_PLAYERS)]
if self.status == 'propose':
proposal_size, _ = self.MISSION_SIZES[self.succeeds + self.fails]
return [ProposeAction(proposal=p) for p in itertools.combinations(range(self.NUM_PLAYERS), r=proposal_size)]
if self.status == 'vote':
return [VoteAction(up=True), VoteAction(up=False)]
if self.status == 'run':
player_role = hidden_state[player]
if player_role in EVIL_ROLES:
return [MissionAction(fail=False), MissionAction(fail=True)]
else:
return [MissionAction(fail=False)]
assert False, "Not sure how we got here"
def get_action(self,
state,
legal_actions,
role_guess=None,
return_all=False):
role_guess = role_guess or random.choice(self.hidden_states)
if state.status == 'vote':
if state.propose_count == 4:
return VoteAction(up=True)
up_vote = role_guess[state.proposer] in GOOD_ROLES and all(
role_guess[p] in GOOD_ROLES for p in state.proposal)
if self.is_evil:
up_vote = not up_vote
return VoteAction(up=up_vote)
if state.status == 'propose' and not self.is_evil:
propose_size = len(legal_actions[0].proposal)
good_players = [
p for p, role in enumerate(role_guess) if role in GOOD_ROLES
]
if return_all:
return [
ProposeAction(proposal=combo)
for combo in itertools.combinations(
good_players, propose_size)
]
random.shuffle(good_players)
return ProposeAction(
proposal=tuple(sorted(good_players[:propose_size])))
if state.status == 'run' and self.is_evil:
return MissionAction(fail=True)
if return_all:
return legal_actions
return random.choice(legal_actions)
def cfr_search_fast(self, state, hidden_state, fails, strategy_probability, t, cache):
if state.is_terminal():
return state.terminal_value(hidden_state)[self.player]
cache_key = (state.as_key(), tuple(fails))
if cache_key in cache:
return cache[cache_key]
if np.random.random() < 0.0001:
print len(cache)
player_statuses = [
[0, 0, 0, 0, 0]
for _ in hidden_state
]
for proposal, _ in fails:
for p in range(len(hidden_state)):
p_on = p in proposal
for player in proposal:
player_statuses[p][player] = max(player_statuses[p][player], 2 if p_on else 1)
moving_players = state.moving_players()
my_move_index = None
moves = [None] * len(moving_players)
for i in range(len(moving_players)):
player = moving_players[i]
if hidden_state[player] not in EVIL_ROLES and state.status == 'run':
moves[i] = MissionAction(fail=False)
continue
if hidden_state[player] != 'assassin' and state.status == 'merlin':
moves[i] = PickMerlinAction(merlin=np.random.choice(len(hidden_state)))
continue
bucket_type, bucket = history_to_bucket(hidden_state, player, [(None, state)], player_statuses[player])
if player == self.player:
my_move_index = i
# moves[i] = [(None, 1.0)]
continue
else:
bucket_data = self.opponent_buckets[player][bucket_type][bucket]
uniform_prob = np.ones(len(bucket_data))/len(bucket_data)
tremble_prob = 1.0/np.sqrt(3 * np.sum(bucket_data) + 1)
if np.sum(bucket_data) == 0:
move_probs = uniform_prob
else:
move_probs = tremble_prob * uniform_prob + (1.0 - tremble_prob) * bucket_data / np.sum(bucket_data)
moves[i] = move_index_to_move(np.random.choice(len(move_probs), p=move_probs), state)
if my_move_index is None:
value = 0.0
new_state, _, observation = state.transition(moves, hidden_state)
if state.status == 'run' and observation > 0:
fails.append((state.proposal, observation))
value = self.cfr_search_fast(new_state, hidden_state, fails, strategy_probability, t, cache)
if state.status == 'run' and observation > 0:
fails.pop()
cache[cache_key] = value
return value
bucket_type, bucket = history_to_bucket(hidden_state, self.player, [(None, state)], player_statuses[self.player])
my_strategy = np.clip(self.cfr_regret[bucket_type][bucket], 0, None)
if np.sum(my_strategy) == 0:
p = np.ones(len(my_strategy))/len(my_strategy)
else:
p = my_strategy / np.sum(my_strategy)
values = np.zeros(len(my_strategy))
for action_index in range(len(values)):
moves[my_move_index] = move_index_to_move(action_index, state)
new_state, _, observation = state.transition(moves, hidden_state)
if state.status == 'run' and observation > 0:
fails.append((state.proposal, observation))
values[action_index] = self.cfr_search_fast(new_state, hidden_state, fails, strategy_probability * p[action_index], t, cache)
if state.status == 'run' and observation > 0:
fails.pop()
strategy_value = np.dot(values, p)
regrets = values - strategy_value
self.cfr_regret[bucket_type][bucket] += regrets * t
cache[cache_key] = strategy_value
return strategy_value
def single_mcts_search(self, state):
history_len = len(self.history)
hidden_state = self.hidden_states[np.random.choice(len(self.hidden_states))]
player_statuses = [
[0, 0, 0, 0, 0]
for _ in hidden_state
]
# Set up player statuses
for proposal, _ in self.fails:
for p in range(len(hidden_state)):
p_on = p in proposal
for player in proposal:
player_statuses[p][player] = max(player_statuses[p][player], 2 if p_on else 1)
visited_nodes = []
chosen_actions = []
value = None
while True:
moves = []
for player in state.moving_players():
if hidden_state[player] not in EVIL_ROLES and state.status == 'run':
moves.append(MissionAction(fail=False))
continue
if hidden_state[player] != 'assassin' and state.status == 'merlin':
moves.append(PickMerlinAction(merlin=np.random.choice(len(hidden_state))))
continue
bucket_type, bucket = history_to_bucket(hidden_state, player, self.history, player_statuses[player])
if player != self.player:
bucket_data = self.opponent_buckets[self.bot_ids[player]][bucket_type][bucket]
uniform_prob = np.ones(len(bucket_data))/len(bucket_data)
tremble_prob = 1.0/np.sqrt(4*np.sum(bucket_data) + 1)
if np.sum(bucket_data) == 0:
move_probs = uniform_prob
else:
move_probs = tremble_prob * uniform_prob + (1.0 - tremble_prob) * bucket_data / np.sum(bucket_data)
move_index = np.random.choice(len(move_probs), p=move_probs)
else:
is_new_node = bucket not in self.my_buckets[bucket_type]
node = self.my_buckets[bucket_type][bucket]
move_index = node.select_move()
if is_new_node:
value = heuristic_value_func(state, hidden_state, self.player)
else:
visited_nodes.append(node)
chosen_actions.append(move_index)
moves.append(move_index_to_move(move_index, state))
if value is not None:
break
state, _, observation = state.transition(moves, hidden_state)
self.history.append((observation, state))
if state.is_terminal():
value = state.terminal_value(hidden_state)[self.player]
break
for node, action in zip(visited_nodes, chosen_actions):
node.choose_counts[action] += 1
node.total_payoffs[action] += 1
self.history = self.history[:history_len]
def handle_round(game, state, hidden_state, bots, round_, stats):
last_proposal = None
for proposal_num in ['1', '2', '3', '4', '5']:
proposal = last_proposal = round_[proposal_num]
assert state.proposer == proposal['proposer']
assert state.propose_count == int(proposal_num) - 1
moves = [ProposeAction(proposal=tuple(sorted(proposal['team'])))]
for player, move in zip(state.moving_players(), moves):
prob = get_prob(state, hidden_state, player, bots[player], move)
stats.append({
'game': game['id'],
'seat': player,
'role': hidden_state[player],
'player': game['players'][player]['player_id'],
'type': 'propose',
'move': ','.join(map(str, sorted(move.proposal))),
'bot': bots[player].__class__.__name__,
'prob': prob,
'num_players': len(hidden_state)
})
state = handle_transition(state, hidden_state, moves, bots)
assert state.status == 'vote'
moves = [
VoteAction(up=(vote == 'Approve')) for vote in proposal['votes']
]
for player, move in zip(state.moving_players(), moves):
prob = get_prob(state, hidden_state, player, bots[player], move)
stats.append({
'game': game['id'],
'seat': player,
'role': hidden_state[player],
'player': game['players'][player]['player_id'],
'type': 'vote',
'move': 'up' if move.up else 'down',
'bot': bots[player].__class__.__name__,
'prob': prob,
'num_players': len(hidden_state)
})
state = handle_transition(state, hidden_state, moves, bots)
if state.status == 'run':
break
secret_votes = sorted(zip(last_proposal['team'], round_['mission']))
moves = [
MissionAction(fail=(vote == "Fail")) for player, vote in secret_votes
]
for player, move in zip(state.moving_players(), moves):
prob = get_prob(state, hidden_state, player, bots[player], move)
stats.append({
'game': game['id'],
'seat': player,
'role': hidden_state[player],
'player': game['players'][player]['player_id'],
'type': 'mission',
'move': 'fail' if move.fail else 'succeed',
'bot': bots[player].__class__.__name__,
'prob': prob,
'num_players': len(hidden_state)
})
state = handle_transition(state, hidden_state, moves, bots)
if state.status == 'merlin':
assert 'findMerlin' in round_
find_merlin = round_['findMerlin']
assert hidden_state[find_merlin['assassin']] == 'assassin'
moves = [
PickMerlinAction(merlin=find_merlin['merlin_guess'])
for _ in hidden_state
]
for player, move in zip(state.moving_players(), moves):
prob = get_prob(state, hidden_state, player, bots[player], move)
if hidden_state[player] == 'assassin':
stats.append({
'game': game['id'],
'seat': player,
'role': hidden_state[player],
'player': game['players'][player]['player_id'],
'type': 'merlin',
'move': move.merlin,
'bot': bots[player].__class__.__name__,
'prob': prob,
'num_players': len(hidden_state)
})
state = handle_transition(state, hidden_state, moves, bots)
return state
def replay_game(game):
roles = game['game_info']['roles']
roles = roles[:1] + roles[1:][::-1]
hidden_state = [PRO_TO_HS[r] for r in roles]
players = game['session_info']['players']
proposer = [
'VHleader' in game['game_info']['voteHistory'][player][0][0]
for player in players
].index(True)
state = AvalonState(
proposer=proposer,
propose_count=0,
succeeds=0,
fails=0,
status='propose',
proposal=None,
game_end=None,
num_players=5
)
yield None, state, hidden_state
while not state.is_terminal():
rnd = state.succeeds + state.fails
if state.status != 'merlin':
proposer = [
'VHleader' in game['game_info']['voteHistory'][player][rnd][state.propose_count]
for player in players
].index(True)
assert proposer == state.proposer, "{} != {}".format(proposer, state.proposer)
if state.status == 'propose':
proposal = tuple(sorted([
players.index(player)
for player in players
if 'VHpicked' in game['game_info']['voteHistory'][player][rnd][state.propose_count]
]))
actions = [ProposeAction(proposal=proposal)]
elif state.status == 'vote':
actions = [
VoteAction(up=(
'VHapprove' in game['game_info']['voteHistory'][player][rnd][state.propose_count]
))
for player in players
]
elif state.status == 'run':
observed_fails = game['game_info']['numFailsHistory'][rnd]
actions = []
for player in state.moving_players():
if hidden_state[player] in set(['merlin', 'servant']):
actions.append(MissionAction(fail=False))
elif observed_fails == 0:
actions.append(MissionAction(fail=False))
else:
actions.append(MissionAction(fail=True))
observed_fails -= 1
assert observed_fails == 0
elif state.status == 'merlin':
shot_player = players.index(game['game_info']['publicData']['roles']['assassinShotUsername'])
actions = [PickMerlinAction(merlin=shot_player) for _ in range(5)]
assert len(actions) == len(state.moving_players())
new_state, _, obs = state.transition(actions, hidden_state)
yield state, new_state, obs
state = new_state
def handle_round(data, state, hidden_state, game, round_):
last_proposal = None
for proposal_num in ['1', '2', '3', '4', '5']:
proposal = last_proposal = round_[proposal_num]
assert state.proposer == proposal['proposer'], "idk"
assert state.propose_count == int(proposal_num) - 1, "idk2"
data.append({
'game': game['id'],
'player': game['players'][proposal['proposer']]['player_id'],
'seat': proposal['proposer'],
'role': hidden_state[proposal['proposer']],
'is_evil': hidden_state[proposal['proposer']] in EVIL_ROLES,
'type': 'propose',
'move': ','.join(map(str, sorted(proposal['team']))),
'propose_count': state.propose_count,
'round': state.fails + state.succeeds,
'succeeds': state.succeeds,
'fails': state.fails,
'propose_has_self': proposal['proposer'] in proposal['team'],
'num_players': len(hidden_state)
})
moves = [ProposeAction(proposal=tuple(sorted(proposal['team'])))]
state, _, _ = state.transition(moves, hidden_state)
assert state.status == 'vote'
for seat, vote in enumerate(proposal['votes']):
data.append({
'game': game['id'],
'player': game['players'][seat]['player_id'],
'seat': seat,
'role': hidden_state[seat],
'is_evil': hidden_state[seat] in EVIL_ROLES,
'type': 'vote',
'move': vote,
'propose_count': state.propose_count,
'round': state.fails + state.succeeds,
'succeeds': state.succeeds,
'fails': state.fails,
'propose_has_self': seat in proposal['team'],
'num_players': len(hidden_state)
})
moves = [VoteAction(up=(vote == 'Approve')) for vote in proposal['votes']]
state, _, _ = state.transition(moves, hidden_state)
if state.status == 'run':
break
secret_votes = sorted(zip(last_proposal['team'], round_['mission']))
for seat, vote in secret_votes:
data.append({
'game': game['id'],
'player': game['players'][seat]['player_id'],
'seat': seat,
'role': hidden_state[seat],
'is_evil': hidden_state[seat] in EVIL_ROLES,
'type': 'mission',
'move': vote,
'propose_count': state.propose_count,
'round': state.fails + state.succeeds,
'succeeds': state.succeeds,
'fails': state.fails,
'propose_has_self': True,
'num_players': len(hidden_state)
})
moves = [MissionAction(fail=(vote == "Fail")) for player, vote in secret_votes]
state, _, _ = state.transition(moves, hidden_state)
if state.status == 'merlin':
assert 'findMerlin' in round_, "wat"
find_merlin = round_['findMerlin']
assert hidden_state[find_merlin['assassin']] == 'assassin', "wat"
moves = [
PickMerlinAction(merlin=find_merlin['merlin_guess'])
for _ in hidden_state
]
seat = hidden_state.index('assassin')
data.append({
'game': game['id'],
'player': game['players'][seat]['player_id'],
'seat': seat,
'role': hidden_state[seat],
'is_evil': hidden_state[seat] in EVIL_ROLES,
'type': 'merlin',
'move': str(find_merlin['merlin_guess']),
'propose_count': state.propose_count,
'round': state.fails + state.succeeds,
'succeeds': state.succeeds,
'fails': state.fails,
'propose_has_self': True,
'num_players': len(hidden_state)
})
state, _, _ = state.transition(moves, hidden_state)
return state