def update_combined(poker_round,model,params):
optimizer = params['model_optimizer']
device = params['device']
state = poker_round['state']
action = poker_round['action']
reward = torch.tensor(poker_round['reward']).unsqueeze(-1).to(device)
betsize_mask = poker_round['betsize_mask']
action_mask = poker_round['action_mask']
# scaled_rewards = scale_rewards(reward,params['min_reward'],params['max_reward'])
## Critic update ##
local_values = model(np.array(state),np.array(action_mask),np.array(betsize_mask))['value']
value_mask = return_value_mask(action)
# TD_error = local_values[value_mask] - reward
# critic_loss = (TD_error**2*0.5).sum()
critic_loss = F.smooth_l1_loss(reward,local_values[value_mask],reduction='sum')
optimizer.zero_grad()
critic_loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), params['gradient_clip'])
optimizer.step()
# Actor update #
actor_out = model(np.array(state),np.array(action_mask),np.array(betsize_mask))
target_values = actor_out['value']
actor_value_mask = return_value_mask(actor_out['action'])
expected_value = (actor_out['action_probs'].view(-1) * target_values.view(-1)).view(actor_value_mask.size()).detach().sum(-1)
advantages = (target_values[actor_value_mask] - expected_value).view(-1)
policy_loss = (-actor_out['action_prob'].view(-1) * advantages).sum()
optimizer.zero_grad()
policy_loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), params['gradient_clip'])
optimizer.step()
return critic_loss.item(),policy_loss.item()
def update_critic_batch(data,local_critic,target_critic,params):
# get the inputs; data is a list of [inputs, targets]
device = params['device']
trajectory, target = data.values()
obs = trajectory['obs'].to(device)
action = trajectory['action'].to(device)
reward = target['reward'].to(device)
betsize_mask = trajectory['betsize_mask'].to(device)
action_mask = trajectory['action_mask'].to(device)
# scaled_rewards = scale_rewards(reward,params['min_reward'],params['max_reward'])
# Critic update
local_values = local_critic(obs)['value']
value_mask = return_value_mask(action)
# TD_error = local_values[value_mask] - reward
# critic_loss = (TD_error**2*0.5).mean()
critic_loss = F.smooth_l1_loss(reward,local_values[value_mask],reduction='sum')
params['critic_optimizer'].zero_grad()
critic_loss.backward()
torch.nn.utils.clip_grad_norm_(local_critic.parameters(), params['gradient_clip'])
params['critic_optimizer'].step()
soft_update(local_critic,target_critic,device,tau=1e-1)
post_local_values = local_critic(obs)['value']
# Eval
table = PrettyTable(["Critic Values","Updated Critic values","Max diff"])
for i in range(local_values.size(0)):
critic_diff = local_values[i].detach().cpu().numpy() - post_local_values[i].detach().cpu().numpy()
critic_diff_max = np.max(np.abs(critic_diff))
table.add_row([local_values.detach()[i],post_local_values.detach()[i],critic_diff_max])
print(table)
table = PrettyTable(["Critic Value","Updated value","Reward","action","Loss"])
for i in range(local_values.size(0)):
table.add_row([local_values.detach()[i][action[i]],post_local_values.detach()[i][action[i]],reward[i],action[i],critic_loss.item()])
print(table)
return critic_loss.item()
def update_actor_batch(data, local_actor, target_actor, target_critic, params):
# get the inputs; data is a list of [inputs, targets]
device = params['device']
trajectory, target = data.values()
state = trajectory['state'].to(device)
obs = trajectory['obs'].to(device)
action = trajectory['action'].to(device)
reward = target['reward'].to(device)
betsize_mask = trajectory['betsize_mask'].to(device)
action_mask = trajectory['action_mask'].to(device)
# scaled_rewards = scale_rewards(reward,params['min_reward'],params['max_reward'])
# Actor update #
target_values = target_critic(obs)['value']
actor_out = local_actor(state, action_mask, betsize_mask)
target_out = target_actor(state, action_mask, betsize_mask)
actor_value_mask = return_value_mask(actor_out['action'])
expected_value = (actor_out['action_probs'].view(-1) *
target_values.view(-1)).view(
actor_value_mask.size()).detach().sum(-1)
advantages = (target_values[actor_value_mask] - expected_value).view(-1)
policy_loss = (-actor_out['action_prob'].view(-1) * advantages).sum()
params['actor_optimizer'].zero_grad()
policy_loss.backward()
# torch.nn.utils.clip_grad_norm_(local_actor.parameters(), params['gradient_clip'])
params['actor_optimizer'].step()
soft_update(local_actor, target_actor, device)
post_actor_out = local_actor(state, action_mask, betsize_mask)
post_target_out = target_actor(state, action_mask, betsize_mask)
# Assert probabilities aren't changing more than x
actor_diff = actor_out['action_probs'].detach().cpu().numpy(
) - post_actor_out['action_probs'].detach().cpu().numpy()
target_diff = target_out['action_probs'].detach().cpu().numpy(
) - post_target_out['action_probs'].detach().cpu().numpy()
actor_diff_max = np.max(np.abs(actor_diff))
target_diff_max = np.max(np.abs(target_diff))
table = PrettyTable(["Critic Q Values", "Action", "Reward", "Policy Loss"])
for i in range(target_values.size(0)):
table.add_row([
target_values.detach()[i], action[i], reward[i],
policy_loss.item()
])
print(table)
table = PrettyTable(
["Actor Probs", "Updated Actor Probs", "Max Actor diff"])
for i in range(actor_out['action_probs'].detach().size(0)):
table.add_row([
actor_out['action_probs'].detach()[i],
post_actor_out['action_probs'].detach()[i], actor_diff_max
])
print(table)
table = PrettyTable(
["Target Actor Probs", "Updated Target Probs", "Max Target diff"])
for i in range(target_out['action_probs'].detach().size(0)):
table.add_row([
target_out['action_probs'].detach()[i],
post_target_out['action_probs'].detach()[i], target_diff_max
])
print(table)
return policy_loss.item()
def update_actor(poker_round, actor, target_actor, target_critic, params):
"""With critic batch updates"""
actor_optimizer = params['actor_optimizer']
device = params['device']
state = poker_round['state']
obs = poker_round['obs']
action = poker_round['action']
reward = poker_round['reward']
betsize_mask = poker_round['betsize_mask']
action_mask = poker_round['action_mask']
# Actor update #
target_values = target_critic(obs)['value']
actor_out = actor(np.array(state), np.array(action_mask),
np.array(betsize_mask))
target_out = target_actor(np.array(state), np.array(action_mask),
np.array(betsize_mask))
actor_value_mask = return_value_mask(actor_out['action'])
expected_value = (actor_out['action_probs'].view(-1) *
target_values.view(-1)).view(
actor_value_mask.size()).detach().sum(-1)
advantages = (target_values[actor_value_mask] - expected_value).view(-1)
policy_loss = (-actor_out['action_prob'].view(-1) * advantages).sum()
actor_optimizer.zero_grad()
policy_loss.backward()
torch.nn.utils.clip_grad_norm_(actor.parameters(), params['gradient_clip'])
actor_optimizer.step()
soft_update(actor, target_actor, device)
post_actor_out = actor(np.array(state), np.array(action_mask),
np.array(betsize_mask))
post_target_out = target_actor(np.array(state), np.array(action_mask),
np.array(betsize_mask))
# Assert probabilities aren't changing more than x
actor_diff = actor_out['action_probs'].detach().cpu().numpy(
) - post_actor_out['action_probs'].detach().cpu().numpy()
target_diff = target_out['action_probs'].detach().cpu().numpy(
) - post_target_out['action_probs'].detach().cpu().numpy()
actor_diff_max = np.max(np.abs(actor_diff))
target_diff_max = np.max(np.abs(target_diff))
table = PrettyTable(["Critic Q Values", "Action", "Reward", "Policy Loss"])
table.add_row([target_values.detach(), action, reward, policy_loss.item()])
print(table)
table = PrettyTable(
["Actor Probs", "Updated Actor Probs", "Max Actor diff"])
table.add_row([
actor_out['action_probs'].detach(),
post_actor_out['action_probs'].detach(), actor_diff_max
])
print(table)
table = PrettyTable(
["Target Actor Probs", "Updated Target Probs", "Max Target diff"])
table.add_row([
target_out['action_probs'].detach(),
post_target_out['action_probs'].detach(), target_diff_max
])
print(table)
def update_actor_critic(poker_round,critic,target_critic,actor,target_actor,params):
critic_optimizer = params['critic_optimizer']
actor_optimizer = params['actor_optimizer']
device = params['device']
state = poker_round['state']
obs = poker_round['obs']
action = poker_round['action']
reward = torch.tensor(poker_round['reward']).unsqueeze(-1).to(device)
betsize_mask = poker_round['betsize_mask']
action_mask = poker_round['action_mask']
## Critic update ##
local_values = critic(obs)['value']
value_mask = return_value_mask(action)
# TD_error = local_values[value_mask] - reward
# critic_loss = (TD_error**2*0.5).mean()
critic_loss = F.smooth_l1_loss(reward,local_values[value_mask],reduction='sum')
critic_optimizer.zero_grad()
critic_loss.backward()
critic_optimizer.step()
soft_update(critic,target_critic,device)
# Actor update #
target_values = target_critic(obs)['value']
actor_out = actor(np.array(state),np.array(action_mask),np.array(betsize_mask))
actor_value_mask = return_value_mask(actor_out['action'])
expected_value = (actor_out['action_probs'].view(-1) * target_values.view(-1)).view(actor_value_mask.size()).detach().sum(-1)
advantages = (target_values[actor_value_mask] - expected_value).view(-1)
policy_loss = (-actor_out['action_prob'].view(-1) * advantages).sum()
actor_optimizer.zero_grad()
policy_loss.backward()
torch.nn.utils.clip_grad_norm_(actor.parameters(), params['gradient_clip'])
actor_optimizer.step()
soft_update(actor,target_actor,device)
# print('\nprobs,prob,actor action,original action',actor_out['action_probs'].detach(),actor_out['action_prob'].detach(),actor_out['action'],action)
# print('\nlocal_values,Q_value',local_values,local_values[value_mask].item())
# print('\ntarget_values,target_Q_value',target_values,target_values[value_mask].item())
# print('\ntarget_values*mask',(actor_out['action_probs'].view(-1) * target_values.view(-1)).view(value_mask.size()))
# print('\nexpected_value',expected_value)
# print('\nadvantages',advantages)
# print('\nreward',reward)
# print('\npolicy_loss',policy_loss)
return critic_loss.item(),policy_loss.item()
def update_critic(poker_round,critic,params):
critic_optimizer = params['critic_optimizer']
device = params['device']
state = poker_round['state']
obs = poker_round['obs']
action = poker_round['action']
reward = torch.tensor(poker_round['reward']).unsqueeze(-1).to(device)
betsize_mask = poker_round['betsize_mask']
action_mask = poker_round['action_mask']
## Critic update ##
local_values = critic(obs)['value']
value_mask = return_value_mask(action)
# TD_error = local_values[value_mask] - reward
# critic_loss = (TD_error**2*0.5).mean()
critic_loss = F.smooth_l1_loss(reward,local_values[value_mask],reduction='sum')
critic_optimizer.zero_grad()
critic_loss.backward()
critic_optimizer.step()
print('local_values[value_mask],reward',local_values[value_mask],reward)
return critic_loss.item()
def update_actor_critic_batch(data,local_actor,local_critic,target_actor,target_critic,params):
# get the inputs; data is a list of [inputs, targets]
device = params['device']
trajectory, target = data.values()
state = trajectory['state']
obs = trajectory['obs']
action = trajectory['action']
reward = target['reward']
betsize_mask = trajectory['betsize_mask']
action_mask = trajectory['action_mask']
# scaled_rewards = scale_rewards(reward,params['min_reward'],params['max_reward'])
# Critic update
local_values = local_critic(obs.to(device))['value']
# target_values = target_critic(obs)['value']
value_mask = return_value_mask(action.to(device))
# TD_error = local_values[value_mask] - reward
# critic_loss = (TD_error**2*0.5).mean()
critic_loss = F.smooth_l1_loss(reward.to(device),local_values[value_mask].to(device),reduction='sum')
params['critic_optimizer'].zero_grad()
critic_loss.backward()
# torch.nn.utils.clip_grad_norm_(local_critic.parameters(), params['gradient_clip'])
params['critic_optimizer'].step()
soft_update(local_critic,target_critic,device,tau=1e-1)
# Actor update #
# post_local_values = local_critic(obs)['value']
post_target_values = target_critic(obs.to(device))['value']
actor_out = local_actor(state.to(device),action_mask.to(device),betsize_mask.to(device))
# target_out = target_actor(state,action_mask,betsize_mask)
actor_value_mask = return_value_mask(actor_out['action'])
expected_value = (actor_out['action_probs'].view(-1) * post_target_values.view(-1)).view(actor_value_mask.size()).detach().sum(-1)
advantages = (post_target_values[actor_value_mask.to(device)] - expected_value).view(-1)
policy_loss = (-actor_out['action_prob'].view(-1) * advantages).sum()
params['actor_optimizer'].zero_grad()
policy_loss.backward()
# torch.nn.utils.clip_grad_norm_(local_actor.parameters(), params['gradient_clip'])
params['actor_optimizer'].step()
soft_update(local_actor,target_actor,device)
# Check action probs and critic vals
# post_actor_out = local_actor(state,action_mask,betsize_mask)
# post_target_out = target_actor(state,action_mask,betsize_mask)
# # Assert probabilities aren't changing more than x
# actor_diff = actor_out['action_probs'].detach().cpu().numpy() - post_actor_out['action_probs'].detach().cpu().numpy()
# target_diff = target_out['action_probs'].detach().cpu().numpy() - post_target_out['action_probs'].detach().cpu().numpy()
# actor_diff_max = np.max(np.abs(actor_diff))
# target_diff_max = np.max(np.abs(target_diff))
# table = PrettyTable(["Critic Values","Updated critic values","Critic Loss"])
# for i in range(post_target_values.size(0)):
# table.add_row([local_values.detach()[i].cpu(),post_local_values.detach()[i].cpu(),critic_loss.item()])
# print(table)
# table = PrettyTable(["Target Critic Values","Updated Target critic values","Action","Reward"])
# for i in range(post_target_values.size(0)):
# table.add_row([target_values.detach()[i].cpu(),post_target_values.detach()[i].cpu(),action[i],reward[i]])
# print(table)
# table = PrettyTable(["Critic Q Values","Action","Reward","Policy Loss"])
# for i in range(post_target_values.size(0)):
# table.add_row([post_target_values.detach().cpu()[i][action[i]],action[i],reward[i],policy_loss.item()])
# print(table)
# table = PrettyTable(["Actor Probs","Updated Actor Probs","Max Actor diff"])
# for i in range(actor_out['action_probs'].detach().size(0)):
# table.add_row([actor_out['action_probs'].detach().cpu()[i],post_actor_out['action_probs'].detach().cpu()[i],actor_diff_max])
# print(table)
# table = PrettyTable(["Target Actor Probs","Updated Target Probs","Max Target diff"])
# for i in range(target_out['action_probs'].detach().size(0)):
# table.add_row([target_out['action_probs'].detach().cpu()[i],post_target_out['action_probs'].detach().cpu()[i],target_diff_max])
# print(table)
return critic_loss.item()