def hir_relabel(self, non_null_future_idx, episode_experience, current_t, replay_buffer, env): """Relabeling trajectories. Args: non_null_future_idx: list of time step where something happens episode_experience: the RL environment current_t: time time step at which the experience is relabeled replay_buffer: the experience replay buffer env: the RL environment Returns: the reset state of the environment """ ep_len = len(episode_experience) s, a, _, s_tp1, _, ag = episode_experience[current_t] if ag: # TODO(ydjiang): k_immediate logic needs improvement for _ in range(self.cfg.k_immediate): ag_text_single = random.choice(ag) g_type = instruction_type(ag_text_single) if self.cfg.paraphrase and g_type != 'unary': ag_text_single = paraphrase_sentence( ag_text_single, delete_color=self.cfg.diverse_scene_content) replay_buffer.add((s, a, env.reward_scale, s_tp1, self.encode_fn(ag_text_single))) if g_type == 'unary' and self.cfg.negate_unary: negative_ag = negate_unary_sentence(ag_text_single) if negative_ag: replay_buffer.add( (s, a, 0.0, s_tp1, self.encode_fn(negative_ag))) # TODO(ydjiang): repeat logit needs improvement goal_count, repeat = 0, 0 while goal_count < self.cfg.future_k and repeat < (ep_len - current_t) * 4: repeat += 1 future = np.random.randint(current_t, ep_len) _, _, _, _, _, ag_future = episode_experience[future] if not ag_future: continue random.shuffle(ag_future) for single_g in ag_future: if instruction_type(single_g) != 'unary': discount = self.cfg.discount**(future - current_t) if self.cfg.paraphrase: single_g = paraphrase_sentence( single_g, delete_color=self.cfg.diverse_scene_content) replay_buffer.add( (s, a, discount * env.reward_scale, s_tp1, self.encode_fn(single_g))) goal_count += 1 break
def hir_relabel(self, non_null_future_idx, episode_experience, current_t, replay_buffer, env): """Relabeling trajectories. Args: non_null_future_idx: list of time step where something happens episode_experience: the RL environment current_t: time time step at which the experience is relabeled replay_buffer: the experience replay buffer env: the RL environment Returns: the reset state of the environment """ s, a, _, s_tp1, g, ag = episode_experience[current_t] if not ag: return for _ in range(min(self.cfg.k_immediate, len(ag) + 1)): ag_text_single = random.choice(ag) g_type = instruction_type(ag_text_single) if self.cfg.paraphrase and g_type != 'unary': ag_text_single = paraphrase_sentence( ag_text_single, delete_color=self.cfg.diverse_scene_content) replay_buffer.add((s, a, env.reward_scale, s_tp1, self.encode_fn(ag_text_single), ag))