class OmahaActor(Network): def __init__(self, seed, nS, nA, nB, params, hidden_dims=(64, 64), activation=F.leaky_relu): super().__init__() self.activation = activation self.nS = nS self.nA = nA self.nB = nB self.combined_output = nA - 2 + nB self.helper_functions = NetworkFunctions(self.nA, self.nB) self.maxlen = params['maxlen'] self.device = params['device'] self.epsilon = params['epsilon'] self.epsilon_weights = params['epsilon_weights'].to(self.device) self.process_input = PreProcessLayer(params) # self.seed = torch.manual_seed(seed) self.state_mapping = params['state_mapping'] self.action_emb = Embedder(Action.UNOPENED, 64) self.betsize_emb = Embedder(self.nB, 64) self.noise = GaussianNoise(self.device) self.emb = 1248 n_heads = 8 depth = 2 # self.attention = EncoderAttention(params['lstm_in'],params['lstm_out']) self.lstm = nn.LSTM(params['lstm_in'], params['lstm_out'], bidirectional=True) self.batchnorm = nn.BatchNorm1d(self.maxlen) # self.blocks = nn.Sequential( # IdentityBlock(hidden_dims=(2560,2560,512),activation=F.leaky_relu), # IdentityBlock(hidden_dims=(512,512,256),activation=F.leaky_relu), # ) self.fc_final = nn.Linear(5120, self.combined_output) def set_device(self, device): self.device = device self.process_input.set_device(device) def forward(self, state, action_mask, betsize_mask, target=False): """ state: B,M,39 """ if not isinstance(state, torch.Tensor): state = torch.tensor(state, dtype=torch.float32).to(self.device) action_mask = torch.tensor(action_mask, dtype=torch.float32).to(self.device) betsize_mask = torch.tensor(betsize_mask, dtype=torch.float32).to(self.device) mask = combined_masks(action_mask, betsize_mask) if target and np.random.random() < self.epsilon: B = state.size(0) # pick random legal move action_masked = self.epsilon_weights * mask action_probs = action_masked / action_masked.sum(-1).unsqueeze(-1) action = action_probs.multinomial(num_samples=1, replacement=False) action_prob = torch.zeros(B, 1) else: out = self.process_input(state) B, M, c = state.size() n_padding = self.maxlen - M if n_padding < 0: h = out[:, -self.maxlen:, :] else: padding = torch.zeros(B, n_padding, out.size(-1)).to(self.device) h = torch.cat((padding, out), dim=1) lstm_out, hidden_states = self.lstm(h) norm = self.batchnorm(lstm_out) # self.attention(out) # blocks_out = self.blocks(lstm_out.view(-1)) t_logits = self.fc_final(norm.view(B, -1)) category_logits = self.noise(t_logits) # skip connection # category_logits += h action_soft = F.softmax(category_logits, dim=-1) action_probs = norm_frequencies(action_soft, mask) m = Categorical(action_probs) action = m.sample() action_prob = m.log_prob(action) previous_action = torch.as_tensor( state[:, -1, self.state_mapping['last_action']]).to(self.device) action_category, betsize_category = self.helper_functions.batch_unwrap_action( action, previous_action) if B > 1: # batch training outputs = { 'action': action, 'action_category': action_category, 'action_prob': action_prob, 'action_probs': action_probs, 'betsize': betsize_category } else: # playing hand outputs = { 'action': action.item(), 'action_category': action_category.item(), 'action_prob': action_prob, 'action_probs': action_probs, 'betsize': betsize_category.item() } return outputs
class OmahaActor(Network): def __init__(self, seed, nS, nA, nB, params, hidden_dims=(64, 64), activation=F.leaky_relu): super().__init__() self.activation = activation self.nS = nS self.nA = nA self.nB = nB self.combined_output = nA - 2 + nB self.helper_functions = NetworkFunctions(self.nA, self.nB) self.maxlen = params['maxlen'] self.device = params['device'] self.process_input = PreProcessLayer(params) # self.seed = torch.manual_seed(seed) self.state_mapping = params['state_mapping'] self.action_emb = Embedder(Action.UNOPENED, 64) self.betsize_emb = Embedder(self.nB, 64) self.noise = GaussianNoise(self.device) self.emb = 1248 n_heads = 8 depth = 2 # self.attention = EncoderAttention(params['lstm_in'],params['lstm_out']) self.lstm = nn.LSTM(params['lstm_in'], params['lstm_out'], bidirectional=True) self.batchnorm = nn.BatchNorm1d(self.maxlen) # self.blocks = nn.Sequential( # IdentityBlock(hidden_dims=(2560,2560,512),activation=F.leaky_relu), # IdentityBlock(hidden_dims=(512,512,256),activation=F.leaky_relu), # ) self.fc_final = nn.Linear(2560, self.combined_output) self.dropout = nn.Dropout(0.5) def forward(self, state, action_mask, betsize_mask): """ state: B,M,39 """ x = state if not isinstance(x, torch.Tensor): x = torch.tensor(x, dtype=torch.float32).to(self.device) action_mask = torch.tensor(action_mask, dtype=torch.float).to(self.device) betsize_mask = torch.tensor(betsize_mask, dtype=torch.float).to(self.device) mask = combined_masks(action_mask, betsize_mask) out = self.process_input(x) B, M, c = out.size() n_padding = self.maxlen - M if n_padding < 0: h = out[:, -self.maxlen:, :] else: padding = torch.zeros(B, n_padding, out.size(-1)).to(self.device) h = torch.cat((padding, out), dim=1) lstm_out, hidden_states = self.lstm(h) norm = self.batchnorm(lstm_out) # self.attention(out) # blocks_out = self.blocks(lstm_out.view(-1)) t_logits = self.fc_final(norm.view(B, -1)) category_logits = self.noise(t_logits) # skip connection # category_logits += h action_soft = F.softmax(category_logits, dim=-1) # if torch.cuda.is_available(): # action_probs = norm_frequencies(action_soft,mask.cuda()) # previous_action = torch.as_tensor(state[:,-1,self.state_mapping['last_action']]).cuda()#.to(self.device) # else: action_probs = norm_frequencies(action_soft, mask) previous_action = torch.as_tensor( state[:, -1, self.state_mapping['last_action']]).to(self.device) m = Categorical(action_probs) action = m.sample() action_category, betsize_category = self.helper_functions.batch_unwrap_action( action, previous_action) if B > 1: # batch training outputs = { 'action': action, 'action_category': action_category, 'action_prob': m.log_prob(action), 'action_probs': action_probs, 'betsize': betsize_category } else: # playing hand outputs = { 'action': action.item(), 'action_category': action_category.item(), 'action_prob': m.log_prob(action), 'action_probs': action_probs, 'betsize': betsize_category.item() } return outputs