def get_b(self, pub_obses, range_idxs, legal_actions_lists, to_np=False):
with torch.no_grad():
range_idxs = torch.tensor(range_idxs, dtype=torch.long, device=self.device)
masks = rl_util.batch_get_legal_action_mask_torch(n_actions=self._env_bldr.N_ACTIONS,
legal_actions_lists=legal_actions_lists,
device=self.device, dtype=torch.float32)
self.eval()
q = self._net(pub_obses=pub_obses, range_idxs=range_idxs, legal_action_masks=masks)
q *= masks
if to_np:
q = q.cpu().numpy()
return q
def get_a_probs(self, pub_obses, range_idxs, legal_actions_lists):
"""
Args:
pub_obses (list): list of np arrays of shape [np.arr([history_len, n_features]), ...)
range_idxs (np.ndarray): array of range_idxs (one for each pub_obs) tensor([2, 421, 58, 912, ...])
legal_actions_lists (list: list of lists. each 2nd level lists contains ints representing legal actions
"""
with torch.no_grad():
masks = rl_util.batch_get_legal_action_mask_torch(
n_actions=self._env_bldr.N_ACTIONS,
legal_actions_lists=legal_actions_lists,
device=self.device)
return self.get_a_probs2(pub_obses=pub_obses,
range_idxs=range_idxs,
legal_action_masks=masks)
def get_a_probs(self, pub_obses, range_idxs, legal_actions_lists, to_np=True):
"""
Args:
pub_obses (list): batch (list) of np arrays of shape [np.arr([history_len, n_features]), ...)
range_idxs (list): batch (list) of range_idxs (one for each pub_obs) [2, 421, 58, 912, ...]
legal_actions_lists (list): batch (list) of lists of integers that represent legal actions
"""
with torch.no_grad():
masks = rl_util.batch_get_legal_action_mask_torch(n_actions=self._env_bldr.N_ACTIONS,
legal_actions_lists=legal_actions_lists,
device=self._device, dtype=torch.float32)
return self.get_a_probs2(pub_obses=pub_obses,
range_idxs=range_idxs,
legal_action_masks=masks,
to_np=to_np)
def get_a_probs_tensor(self, pub_obses, range_idxs, legal_actions_lists,
hand_info):
## with torch.no_grad(): <--- this function is used in training
masks = rl_util.batch_get_legal_action_mask_torch(
n_actions=self._env_bldr.N_ACTIONS,
legal_actions_lists=legal_actions_lists,
device=self.device)
masks = masks.view(1, -1)
pred = self._net(pub_obses=pub_obses,
range_idxs=torch.from_numpy(range_idxs).to(
dtype=torch.long, device=self.device),
legal_action_masks=masks,
hand_info=hand_info)
return nnf.softmax(
pred, dim=-1).cpu() #.numpy() removed to stay in Tensor form
def select_br_a(self, pub_obses, range_idxs, legal_actions_lists, explore=False):
if explore and (np.random.random() < self._eps):
return np.array(
[legal_actions[np.random.randint(len(legal_actions))] for legal_actions in legal_actions_lists]
)
with torch.no_grad():
self.eval()
range_idxs = torch.tensor(range_idxs, dtype=torch.long, device=self.device)
q = self._net(pub_obses=pub_obses, range_idxs=range_idxs,
legal_action_masks=rl_util.batch_get_legal_action_mask_torch(
n_actions=self._env_bldr.N_ACTIONS,
legal_actions_lists=legal_actions_lists,
device=self.device,
dtype=torch.float32)).cpu().numpy()
for b in range(q.shape[0]):
illegal_actions = [i for i in self._n_actions_arranged if i not in legal_actions_lists[b]]
if len(illegal_actions) > 0:
illegal_actions = np.array(illegal_actions)
q[b, illegal_actions] = -1e20
return np.argmax(q, axis=1)