class API(object):
def __init__(self):
self.increment_position = {'SB': 'BB', 'BB': 'SB'}
self.seed = 1458
self.connect()
self.game_object = pdt.Globals.GameTypeDict[pdt.GameTypes.OMAHAHI]
self.config = Config()
self.env_params = {
'game': pdt.GameTypes.OMAHAHI,
'betsizes': self.game_object.rule_params['betsizes'],
'bet_type': self.game_object.rule_params['bettype'],
'n_players': 2,
'pot': 1,
'stacksize': 10, #self.game_object.state_params['stacksize'],
'cards_per_player':
self.game_object.state_params['cards_per_player'],
'starting_street':
pdt.Street.FLOP, #self.game_object.starting_street,
'global_mapping': self.config.global_mapping,
'state_mapping': self.config.state_mapping,
'obs_mapping': self.config.obs_mapping,
'shuffle': True
}
self.env = Poker(self.env_params)
self.network_params = self.instantiate_network_params()
self.actor = OmahaActor(self.seed, self.env.state_space,
self.env.action_space, self.env.betsize_space,
self.network_params)
self.critic = OmahaObsQCritic(self.seed, self.env.state_space,
self.env.action_space,
self.env.betsize_space,
self.network_params)
self.load_model(self.actor, self.config.production_actor)
self.load_model(self.critic, self.config.production_critic)
self.player = {'name': None, 'position': 'BB'}
self.reset_trajectories()
def reset_trajectories(self):
self.trajectories = defaultdict(lambda: [])
self.trajectory = defaultdict(
lambda: {
'states': [],
'obs': [],
'betsize_masks': [],
'action_masks': [],
'actions': [],
'action_category': [],
'action_probs': [],
'action_prob': [],
'betsize': [],
'rewards': [],
'value': []
})
def instantiate_network_params(self):
device = 'cpu'
network_params = copy.deepcopy(self.config.network_params)
network_params['maxlen'] = 10
network_params['device'] = device
return network_params
def load_model(self, model, path):
if os.path.isfile(path):
model.load_state_dict(load(path, map_location=D('cpu')))
set_grad_enabled(False)
else:
raise ValueError('File does not exist')
def connect(self):
client = MongoClient('localhost', 27017, maxPoolSize=10000)
self.db = client.baseline
def update_player_name(self, name: str):
"""updates player name"""
self.player['name'] = name
def update_player_position(self, position):
self.player['position'] = position
def insert_model_outputs(self, model_outputs, action_mask):
outputs_json = {
'action': model_outputs['action'],
'action_category': model_outputs['action_category'],
'betsize': model_outputs['betsize'],
'action_prob':
model_outputs['action_prob'].detach().numpy().tolist(),
'action_probs':
model_outputs['action_probs'].detach().numpy().tolist(),
'value': model_outputs['value'].detach().numpy().tolist(),
'action_mask': action_mask.tolist(),
'player': self.player['name']
}
self.db['bot_data'].insert_one(outputs_json)
def insert_into_db(self, training_data: dict):
"""
stores player data in the player_stats collection.
takes trajectories and inserts them into db for data analysis and learning.
"""
stats_json = {
'game': self.env.game,
'player': self.player['name'],
'reward': training_data[self.player['position']][0]['rewards'][0],
'position': self.player['position'],
}
self.db['player_stats'].insert_one(stats_json)
keys = training_data.keys()
positions = [position for position in keys if position in ['SB', 'BB']]
for position in positions:
for i, poker_round in enumerate(training_data[position]):
states = poker_round['states']
observations = poker_round['obs']
actions = poker_round['actions']
action_prob = poker_round['action_prob']
action_probs = poker_round['action_probs']
action_categories = poker_round['action_category']
betsize_masks = poker_round['betsize_masks']
action_masks = poker_round['action_masks']
rewards = poker_round['rewards']
betsizes = poker_round['betsize']
values = poker_round['value']
assert (isinstance(rewards, list))
assert (isinstance(actions, list))
assert (isinstance(action_prob, list))
assert (isinstance(action_probs, list))
assert (isinstance(states, list))
assert (isinstance(values, list))
for step, state in enumerate(states):
state_json = {
'game': self.env.game,
'player': self.player['name'],
'poker_round': step,
'state': state.tolist(),
'action_probs': action_probs[step].tolist(),
'action_prob': action_prob[step].tolist(),
'action': actions[step],
'action_category': action_categories[step],
'betsize_mask': betsize_masks[step].tolist(),
'action_mask': action_masks[step].tolist(),
'betsize': betsizes[step],
'reward': rewards[step],
'value': values[step].tolist()
}
self.db['game_data'].insert_one(state_json)
def return_model_outputs(self):
query = {'player': self.player['name']}
player_data = self.db['bot_data'].find(query).sort('_id', -1)
action_probs = []
values = []
action_mask = []
for result in player_data:
action_probs.append(np.array(result['action_probs']))
values.append(np.array(result['value']))
action_mask.append(np.array(result['action_mask']))
break
if action_probs:
action_probs = action_probs[0]
values = values[0]
action_mask = action_mask[0]
if np.sum(action_probs) > 0:
action_probs *= action_mask
action_probs /= np.sum(action_probs)
# scale values
if np.max(np.abs(values)) > 0:
values *= action_mask
values /= self.env_params['stacksize'] + self.env_params['pot']
model_outputs = {
'action_probs': action_probs.tolist(),
'q_values': [values.tolist()]
}
else:
model_outputs = {
'action_probs': [0] * self.env.action_space,
'q_values': [0] * self.env.action_space
}
print(model_outputs)
print(action_mask)
return model_outputs
def return_player_stats(self):
"""Returns dict of current player stats against the bot."""
query = {'player': self.player['name']}
# projection ={'reward':1,'hand_num':1,'_id':0}
player_data = self.db['player_stats'].find(query)
total_hands = self.db['player_stats'].count_documents(query)
results = []
position_results = {'SB': 0, 'BB': 0}
# total_hands = 0
for result in player_data:
results.append(result['reward'])
position_results[result['position']] += result['reward']
bb_per_hand = sum(results) / total_hands if total_hands > 0 else 0
sb_bb_per_hand = position_results[
'SB'] / total_hands if total_hands > 0 else 0
bb_bb_per_hand = position_results[
'BB'] / total_hands if total_hands > 0 else 0
player_stats = {
'results': sum(results),
'bb_per_hand': round(bb_per_hand, 2),
'total_hands': total_hands,
'SB': round(sb_bb_per_hand, 2),
'BB': round(bb_bb_per_hand, 2),
}
return player_stats
def parse_env_outputs(self, state, action_mask, betsize_mask, done):
"""Wraps state and passes to frontend. Can be the dummy last state. In which case hero mappings are reversed."""
reward = state[:, -1][:, self.env.state_mapping[
'hero_stacksize']] - self.env.starting_stack
# cards go in a list
hero = self.env.players[self.player['position']]
villain = self.env.players[self.increment_position[
self.player['position']]]
state_object = {
'history':
state.tolist(),
'betsizes':
self.env.betsizes.tolist(),
'mapping':
self.env.state_mapping,
'current_player':
pdt.Globals.POSITION_MAPPING[self.env.current_player],
'hero_stack':
hero.stack,
'hero_position':
pdt.Globals.POSITION_MAPPING[hero.position],
'hero_cards':
flatten(hero.hand),
'hero_street_total':
hero.street_total,
'pot':
float(state[:, -1][:, self.env.state_mapping['pot']][0]),
'board_cards':
state[:, -1][:, self.env.state_mapping['board']][0].tolist(),
'villain_stack':
villain.stack,
'villain_position':
pdt.Globals.POSITION_MAPPING[villain.position],
'villain_cards':
flatten(villain.hand),
'villain_street_total':
villain.street_total,
'last_action':
int(state[:, -1][:, self.env.state_mapping['last_action']][0]),
'last_betsize':
float(state[:, -1][:, self.env.state_mapping['last_betsize']][0]),
'last_position':
int(state[:, -1][:, self.env.state_mapping['last_position']][0]),
'last_aggressive_action':
int(state[:, -1]
[:, self.env.state_mapping['last_aggressive_action']][0]),
'last_aggressive_betsize':
float(state[:, -1]
[:, self.env.state_mapping['last_aggressive_betsize']][0]),
'last_aggressive_position':
int(state[:, -1]
[:, self.env.state_mapping['last_aggressive_position']][0]),
'done':
done,
'action_mask':
action_mask.tolist(),
'betsize_mask':
betsize_mask.tolist(),
'street':
int(state[:, -1][:, self.env.state_mapping['street']][0]),
'blind':
bool(state[:, -1][:, self.env.state_mapping['blind']][0])
}
outcome_object = {
'player1_reward': hero.stack - self.env.starting_stack,
'player1_hand': flatten(hero.hand),
'player2_reward': villain.stack - self.env.starting_stack,
'player2_hand': flatten(villain.hand),
'player1_handrank': hero.handrank,
'player2_handrank': villain.handrank
}
json_obj = {'state': state_object, 'outcome': outcome_object}
return json.dumps(json_obj)
def store_state(self, state, obs, action_mask, betsize_mask):
cur_player = self.env.current_player
self.trajectory[cur_player]['states'].append(copy.copy(state))
self.trajectory[cur_player]['action_masks'].append(
copy.copy(action_mask))
self.trajectory[cur_player]['betsize_masks'].append(
copy.copy(betsize_mask))
def store_actions(self, actor_outputs):
cur_player = self.env.current_player
self.trajectory[cur_player]['actions'].append(actor_outputs['action'])
self.trajectory[cur_player]['action_category'].append(
actor_outputs['action_category'])
self.trajectory[cur_player]['action_prob'].append(
actor_outputs['action_prob'])
self.trajectory[cur_player]['action_probs'].append(
actor_outputs['action_probs'])
self.trajectory[cur_player]['betsize'].append(actor_outputs['betsize'])
self.trajectory[cur_player]['value'].append(actor_outputs['value'])
def query_bot(self, state, obs, action_mask, betsize_mask, done):
while self.env.current_player != self.player['position'] and not done:
actor_outputs = self.actor(state, action_mask, betsize_mask)
critic_outputs = self.critic(obs)
actor_outputs['value'] = critic_outputs['value']
self.insert_model_outputs(actor_outputs, action_mask)
self.store_actions(actor_outputs)
state, obs, done, action_mask, betsize_mask = self.env.step(
actor_outputs)
if not done:
self.store_state(state, obs, action_mask, betsize_mask)
return state, obs, done, action_mask, betsize_mask
def reset(self):
assert self.player['name'] is not None
assert isinstance(self.player['position'], str)
self.reset_trajectories()
self.update_player_position(
self.increment_position[self.player['position']])
state, obs, done, action_mask, betsize_mask = self.env.reset()
self.store_state(state, obs, action_mask, betsize_mask)
if self.env.current_player != self.player['position'] and not done:
state, obs, done, action_mask, betsize_mask = self.query_bot(
state, obs, action_mask, betsize_mask, done)
assert self.env.current_player == self.player['position']
return self.parse_env_outputs(state, action_mask, betsize_mask, done)
def step(self, action: str, betsize: float):
"""Maps action + betsize -> to a flat action category"""
assert self.player['name'] is not None
assert isinstance(self.player['position'], str)
if isinstance(betsize, str):
betsize = float(betsize)
action_type = pdt.Globals.SERVER_ACTION_DICT[action]
flat_action_category, betsize_category = self.env.convert_to_category(
action_type, betsize)
assert isinstance(flat_action_category, int)
player_outputs = {
'action':
flat_action_category,
'action_category':
action_type,
'betsize':
betsize_category,
'action_prob':
np.array([0]),
'action_probs':
np.zeros(self.env.action_space + self.env.betsize_space - 2),
'value':
np.zeros(self.env.action_space + self.env.betsize_space - 2)
}
self.store_actions(player_outputs)
state, obs, done, action_mask, betsize_mask = self.env.step(
player_outputs)
if not done:
self.store_state(state, obs, action_mask, betsize_mask)
if self.env.current_player != self.player['position']:
state, obs, done, action_mask, betsize_mask = self.query_bot(
state, obs, action_mask, betsize_mask, done)
if done:
rewards = self.env.player_rewards()
for position in self.trajectory.keys():
N = len(self.trajectory[position]['betsize_masks'])
self.trajectory[position]['rewards'] = [rewards[position]] * N
self.trajectories[position].append(self.trajectory[position])
self.insert_into_db(self.trajectories)
return self.parse_env_outputs(state, action_mask, betsize_mask, done)
@property
def current_player(self):
return self.player
def testEnvCategoryMapping(self):
params = copy.deepcopy(self.env_params)
params['stacksize'] = 50
params['n_players'] = 2
params['starting_street'] = pdt.Street.PREFLOP
params['pot'] = 0
env = Poker(params)
state, obs, done, mask, betsize_mask = env.reset()
assert env.convert_to_category(pdt.NetworkActions.RAISE, 3)[0] == 4
assert env.convert_to_category(pdt.NetworkActions.RAISE, 2)[0] == 3
assert env.convert_to_category(pdt.NetworkActions.CALL, 0.5)[0] == 2
assert env.convert_to_category(pdt.NetworkActions.CHECK, 0)[0] == 0
state, obs, done, mask, betsize_mask = env.step(ACTION_RAISE)
assert env.convert_to_category(pdt.NetworkActions.RAISE, 9)[0] == 4
assert env.convert_to_category(pdt.NetworkActions.RAISE, 5)[0] == 3
assert env.convert_to_category(pdt.NetworkActions.CALL, 2)[0] == 2
assert env.convert_to_category(pdt.NetworkActions.CHECK, 0)[0] == 0
state, obs, done, mask, betsize_mask = env.step(ACTION_CALL)
assert env.convert_to_category(pdt.NetworkActions.BET, 6)[0] == 4
assert env.convert_to_category(pdt.NetworkActions.BET, 3)[0] == 3
assert env.convert_to_category(pdt.NetworkActions.FOLD, 0)[0] == 1
state, obs, done, mask, betsize_mask = env.step(ACTION_BET)
assert env.convert_to_category(pdt.NetworkActions.RAISE, 24)[0] == 4
assert env.convert_to_category(pdt.NetworkActions.RAISE, 12)[0] == 3
assert env.convert_to_category(pdt.NetworkActions.CALL, 6)[0] == 2
assert env.convert_to_category(pdt.NetworkActions.FOLD, 0)[0] == 1
state, obs, done, mask, betsize_mask = env.step(ACTION_RAISE)
assert env.convert_to_category(pdt.NetworkActions.RAISE, 47)[0] == 4
assert env.convert_to_category(pdt.NetworkActions.RAISE, 42)[0] == 3
assert env.convert_to_category(pdt.NetworkActions.CALL, 18)[0] == 2
assert env.convert_to_category(pdt.NetworkActions.FOLD, 0)[0] == 1
del env
params['stacksize'] = 3
params['n_players'] = 2
params['starting_street'] = pdt.Street.PREFLOP
params['pot'] = 0
env = Poker(params)
state, obs, done, mask, betsize_mask = env.reset()
assert env.convert_to_category(pdt.NetworkActions.RAISE, 3)[0] == 4
print('check', env.convert_to_category(pdt.NetworkActions.RAISE, 2)[0])
assert env.convert_to_category(pdt.NetworkActions.RAISE, 2)[0] == 3
assert env.convert_to_category(pdt.NetworkActions.CALL, 0)[0] == 2
assert env.convert_to_category(pdt.NetworkActions.FOLD, 0)[0] == 1