def command_generation_teacher_force(self, observation_strings, task_desc_strings, target_strings):
input_target_strings = [" ".join(["[CLS]"] + item.split()) for item in target_strings]
output_target_strings = [" ".join(item.split() + ["[SEP]"]) for item in target_strings]
input_obs = self.get_word_input(observation_strings)
h_obs, obs_mask = self.encode(observation_strings, use_model="online")
h_td, td_mask = self.encode(task_desc_strings, use_model="online")
aggregated_obs_representation = self.online_net.aggretate_information(h_obs, obs_mask, h_td, td_mask) # batch x obs_length x hid
input_target = self.get_word_input(input_target_strings)
ground_truth = self.get_word_input(output_target_strings) # batch x target_length
target_mask = compute_mask(input_target) # mask of ground truth should be the same
pred = self.online_net.decode(input_target, target_mask, aggregated_obs_representation, obs_mask, None, input_obs) # batch x target_length x vocab
batch_loss = NegativeLogLoss(pred * target_mask.unsqueeze(-1), ground_truth, target_mask, smoothing_eps=self.smoothing_eps)
loss = torch.mean(batch_loss)
if loss is None:
return None
# Backpropagate
self.online_net.zero_grad()
self.optimizer.zero_grad()
loss.backward()
# `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
torch.nn.utils.clip_grad_norm_(self.online_net.parameters(), self.clip_grad_norm)
self.optimizer.step() # apply gradients
return to_np(loss)
def admissible_commands_recurrent_teacher_force(self, seq_observation_strings, seq_task_desc_strings, seq_action_candidate_list, seq_target_indices, contains_first_step=False):
loss_list = []
previous_dynamics = None
h_td, td_mask = self.encode(seq_task_desc_strings[0], use_model="online")
for step_no in range(self.dagger_replay_sample_history_length):
expert_indicies = to_pt(np.array(seq_target_indices[step_no]), enable_cuda=self.use_cuda, type='long')
h_obs, obs_mask = self.encode(seq_observation_strings[step_no], use_model="online")
action_scores, _, current_dynamics = self.action_scoring(seq_action_candidate_list[step_no], h_obs, obs_mask, h_td, td_mask,
previous_dynamics, use_model="online")
previous_dynamics = current_dynamics
if (not contains_first_step) and step_no < self.dagger_replay_sample_update_from:
previous_dynamics = previous_dynamics.detach()
continue
# softmax and cross-entropy
loss = self.cross_entropy_loss(action_scores, expert_indicies)
loss_list.append(loss)
loss = torch.stack(loss_list).mean()
# Backpropagate
self.online_net.zero_grad()
self.optimizer.zero_grad()
loss.backward()
# `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
torch.nn.utils.clip_grad_norm_(self.online_net.parameters(), self.clip_grad_norm)
self.optimizer.step() # apply gradients
return to_np(loss)
def command_generation_teacher_force(self, observation_feats,
task_desc_strings, target_strings):
input_target_strings = [
" ".join(["[CLS]"] + item.split()) for item in target_strings
]
output_target_strings = [
" ".join(item.split() + ["[SEP]"]) for item in target_strings
]
batch_size = len(observation_feats)
aggregated_obs_feat = self.aggregate_feats_seq(observation_feats)
h_obs = self.online_net.vision_fc(aggregated_obs_feat)
h_td, td_mask = self.encode(task_desc_strings, use_model="online")
h_td_mean = self.online_net.masked_mean(h_td, td_mask).unsqueeze(1)
h_obs = h_obs.to(h_td_mean.device)
vision_td = torch.cat((h_obs, h_td_mean),
dim=1) # batch x k boxes x hi
vision_td_mask = torch.ones(
(batch_size,
h_obs.shape[1] + h_td_mean.shape[1])).to(h_td_mean.device)
input_target = self.get_word_input(input_target_strings)
ground_truth = self.get_word_input(
output_target_strings) # batch x target_length
target_mask = compute_mask(
input_target) # mask of ground truth should be the same
pred = self.online_net.vision_decode(
input_target, target_mask, vision_td, vision_td_mask,
None) # batch x target_length x vocab
batch_loss = NegativeLogLoss(pred * target_mask.unsqueeze(-1),
ground_truth,
target_mask,
smoothing_eps=self.smoothing_eps)
loss = torch.mean(batch_loss)
if loss is None:
return None, None
# Backpropagate
self.online_net.zero_grad()
self.optimizer.zero_grad()
loss.backward()
# `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
torch.nn.utils.clip_grad_norm_(self.online_net.parameters(),
self.clip_grad_norm)
self.optimizer.step() # apply gradients
return to_np(pred), to_np(loss)
def command_generation_greedy_generation(self, observation_feats,
task_desc_strings,
previous_dynamics):
with torch.no_grad():
batch_size = len(observation_feats)
aggregated_obs_feat = self.aggregate_feats_seq(observation_feats)
h_obs = self.online_net.vision_fc(aggregated_obs_feat)
h_td, td_mask = self.encode(task_desc_strings, use_model="online")
h_td_mean = self.online_net.masked_mean(h_td, td_mask).unsqueeze(1)
h_obs = h_obs.to(h_td_mean.device)
vision_td = torch.cat((h_obs, h_td_mean),
dim=1) # batch x k boxes x hid
vision_td_mask = torch.ones(
(batch_size,
h_obs.shape[1] + h_td_mean.shape[1])).to(h_td_mean.device)
if self.recurrent:
averaged_vision_td_representation = self.online_net.masked_mean(
vision_td, vision_td_mask)
current_dynamics = self.online_net.rnncell(
averaged_vision_td_representation, previous_dynamics
) if previous_dynamics is not None else self.online_net.rnncell(
averaged_vision_td_representation)
else:
current_dynamics = None
# greedy generation
input_target_list = [[self.word2id["[CLS]"]]
for i in range(batch_size)]
eos = np.zeros(batch_size)
for _ in range(self.max_target_length):
input_target = copy.deepcopy(input_target_list)
input_target = pad_sequences(
input_target, maxlen=max_len(input_target)).astype('int32')
input_target = to_pt(input_target, self.use_cuda)
target_mask = compute_mask(
input_target) # mask of ground truth should be the same
pred = self.online_net.vision_decode(
input_target, target_mask, vision_td, vision_td_mask,
current_dynamics) # batch x target_length x vocab
# pointer softmax
pred = to_np(pred[:, -1]) # batch x vocab
pred = np.argmax(pred, -1) # batch
for b in range(batch_size):
new_stuff = [pred[b]] if eos[b] == 0 else []
input_target_list[b] = input_target_list[b] + new_stuff
if pred[b] == self.word2id["[SEP]"]:
eos[b] = 1
if np.sum(eos) == batch_size:
break
res = [self.tokenizer.decode(item) for item in input_target_list]
res = [
item.replace("[CLS]", "").replace("[SEP]", "").strip()
for item in res
]
res = [item.replace(" in / on ", " in/on ") for item in res]
return res, current_dynamics
def choose_softmax_action(self, action_rank, action_mask=None):
action_rank = action_rank - torch.min(action_rank, -1, keepdim=True)[0] + 1e-2 # minus the min value, so that all values are non-negative
if action_mask is not None:
assert action_mask.size() == action_rank.size(), (action_mask.size().shape, action_rank.size())
action_rank = action_rank * action_mask
pred_softmax = torch.log_softmax(action_rank, dim=1)
action_indices = torch.argmax(pred_softmax, -1) # batch
return pred_softmax, to_np(action_indices)
def command_generation_recurrent_teacher_force(self, seq_observation_strings, seq_task_desc_strings, seq_target_strings, contains_first_step=False):
loss_list = []
previous_dynamics = None
h_td, td_mask = self.encode(seq_task_desc_strings[0], use_model="online")
for step_no in range(self.dagger_replay_sample_history_length):
input_target_strings = [" ".join(["[CLS]"] + item.split()) for item in seq_target_strings[step_no]]
output_target_strings = [" ".join(item.split() + ["[SEP]"]) for item in seq_target_strings[step_no]]
input_obs = self.get_word_input(seq_observation_strings[step_no])
h_obs, obs_mask = self.encode(seq_observation_strings[step_no], use_model="online")
aggregated_obs_representation = self.online_net.aggretate_information(h_obs, obs_mask, h_td, td_mask) # batch x obs_length x hid
averaged_representation = self.online_net.masked_mean(aggregated_obs_representation, obs_mask) # batch x hid
current_dynamics = self.online_net.rnncell(averaged_representation, previous_dynamics) if previous_dynamics is not None else self.online_net.rnncell(averaged_representation)
input_target = self.get_word_input(input_target_strings)
ground_truth = self.get_word_input(output_target_strings) # batch x target_length
target_mask = compute_mask(input_target) # mask of ground truth should be the same
pred = self.online_net.decode(input_target, target_mask, aggregated_obs_representation, obs_mask, current_dynamics, input_obs) # batch x target_length x vocab
previous_dynamics = current_dynamics
if (not contains_first_step) and step_no < self.dagger_replay_sample_update_from:
previous_dynamics = previous_dynamics.detach()
continue
batch_loss = NegativeLogLoss(pred * target_mask.unsqueeze(-1), ground_truth, target_mask, smoothing_eps=self.smoothing_eps)
loss = torch.mean(batch_loss)
loss_list.append(loss)
loss = torch.stack(loss_list).mean()
if loss is None:
return None
# Backpropagate
self.online_net.zero_grad()
self.optimizer.zero_grad()
loss.backward()
# `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
torch.nn.utils.clip_grad_norm_(self.online_net.parameters(), self.clip_grad_norm)
self.optimizer.step() # apply gradients
return to_np(loss)
def admissible_commands_teacher_force(self, observation_strings, task_desc_strings, action_candidate_list, target_indices):
expert_indicies = to_pt(np.array(target_indices), enable_cuda=self.use_cuda, type='long')
h_obs, obs_mask = self.encode(observation_strings, use_model="online")
h_td, td_mask = self.encode(task_desc_strings, use_model="online")
action_scores, _, _ = self.action_scoring(action_candidate_list,
h_obs, obs_mask,
h_td, td_mask,
None,
use_model="online")
# softmax and cross-entropy
loss = self.cross_entropy_loss(action_scores, expert_indicies)
# Backpropagate
self.online_net.zero_grad()
self.optimizer.zero_grad()
loss.backward()
# `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
torch.nn.utils.clip_grad_norm_(self.online_net.parameters(), self.clip_grad_norm)
self.optimizer.step() # apply gradients
return to_np(loss)