def _new_episode(self, args, episode, glove=None, protos=None, pre_metadata=None): """ New navigation episode. """ if episode == None: return None scene = episode["scene"] if self._env is None: self._env = Environment( offline_data_dir=args.offline_data_dir, use_offline_controller=True, grid_size=args.grid_size, images_file_name=args.images_file_name, local_executable_path=args.local_executable_path, rotate_by=args.rotate_by, state_decimal=args.state_decimal, pinned_scene=args.pinned_scene, pre_metadata=pre_metadata, actions=self.actions) self._env.start(scene) else: self._env.reset(scene) self.environment.controller.state = ThorAgentState( **episode['initial_position'], rotation=episode['initial_orientation'], horizon=0, state_decimal=args.state_decimal) self.task_data = [episode["object_id"]] object_type = episode["object_type"] self.target_object = object_type self.difficulty = episode["difficulty"] self.episode_id = episode['id'] if args.glove_file != "": self.glove_embedding = toFloatTensor( glove.glove_embeddings[object_type][:], self.gpu_id) if args.proto_file != "": self.prototype = toFloatTensor( protos.protos[object_type.lower()][:], self.gpu_id) if args.verbose: print("Scene", scene, "Navigating towards:", self.target_object) return scene
def _new_episode(self, args, episode, glove, optimal_act): """ New navigation episode. """ scene = episode["scene"] if self._env is None: self._env = Environment( offline_data_dir=args.offline_data_dir, use_offline_controller=True, grid_size=0.25, images_file_name=args.images_file_name, local_executable_path=args.local_executable_path, ) self._env.start(scene) else: self._env.reset(scene) # self.environment.controller.state = episode["state"] y = 0.9009995 x, z, hor, rot = episode["state"].split('|') self.environment.controller.state = ThorAgentState( float(x), float(y), float(z), float(hor), float(rot)) self.task_data = episode["task_data"] self.target_object = episode["goal_object_type"] if args.verbose: print("Scene", scene, "Navigating towards:", self.target_object) # self.glove_embedding = gpuify(episode["glove_embedding"], self.gpu_id) glove = glove[self.environment.controller.scene_name] self.glove_embedding = None if optimal_act is not None: self.optimal_actions = optimal_act[ self.environment.controller.scene_name][self.task_data[0]] else: self.optimal_actions = None init_pos = '{}|{}|{}|{}'.format( # self.environment.controller.scene_name, self.environment.controller.state.position()['x'], self.environment.controller.state.position()['z'], self.environment.controller.state.rotation, self.environment.controller.state.horizon) # init_pos = self.environment.controller.state target_embedding_array = np.zeros((len(CLASSES), 1)) target_embedding_array[CLASSES.index(self.target_object)] = 1 glove_embedding_tensor = np.concatenate( (glove[init_pos], target_embedding_array), axis=1) self.glove_embedding = toFloatTensor(glove_embedding_tensor, self.gpu_id) self.glove_reader = glove return True
def _new_episode(self, args, scenes, possible_targets, targets=None, room=None, keep_obj=False, glove=None): """ New navigation episode. """ scene = random.choice(scenes) self.room = room if self._env is None: self._env = Environment( offline_data_dir=args.offline_data_dir, use_offline_controller=True, grid_size=0.25, images_file_name=args.images_file_name, local_executable_path=args.local_executable_path, ) self._env.start(scene) else: self._env.reset(scene) # Randomize the start location. start_state = self._env.randomize_agent_location() objects = self._env.all_objects() visible_objects = [obj.split("|")[0] for obj in objects] intersection = [obj for obj in visible_objects if obj in targets] self.task_data = [] idx = random.randint(0, len(intersection) - 1) goal_object_type = intersection[idx] self.target_object = goal_object_type for id_ in objects: type_ = id_.split("|")[0] if goal_object_type == type_: self.task_data.append(id_) child_object = self.task_data[0].split("|")[0] #print('room is ', self.room) try: self.target_parents = c2p_prob[self.room][child_object] except: self.target_parents = None if args.verbose: print("Scene", scene, "Navigating towards:", goal_object_type) self.glove_embedding = None self.glove_embedding = toFloatTensor( glove.glove_embeddings[goal_object_type][:], self.gpu_id)
def _new_episode(self, args, episode): """ New navigation episode. """ scene = episode["scene"] if "physics" in scene: scene = scene[:-8] if self._env is None: self._env = Environment( offline_data_dir=args.offline_data_dir, use_offline_controller=True, grid_size=0.25, images_file_name=args.images_file_name, local_executable_path=args.local_executable_path, ) self._env.start(scene) else: self._env.reset(scene) self.environment.controller.state = episode["state"] self.task_data = episode["task_data"] self.target_object = episode["goal_object_type"] # with open(metadata_dir + scene + '/visible_object_map.json', 'r')as f : # meta_data = json.load(f) # self.task_data = [] # for k, v in meta_data.items(): # obj = k.split('|')[0] # if self.target_object == obj: # self.task_data.append(k) # regex = re.compile('FloorPlan([0-9]*)') # num = int(regex.findall(scene)[0]) # ind = int(num / 100) # if ind > 0: # ind -= 1 # room = rooms[ind] self.room = episode["room"] try: self.target_parents = c2p_prob[self.room][self.target_object] except KeyError: print("finding", self.target_object, 'in', self.room) if args.verbose: print("Scene", scene, "Navigating towards:", self.target_object) if args.vis: log = open('saved_action_' + args.results_json[:-5] + '.log', "a+") sys.stdout = log print("Scene", scene, "Navigating towards:", self.target_object) print(self.environment.controller.state, self.environment.controller.state.y) self.glove_embedding = toFloatTensor(episode["glove_embedding"], self.gpu_id) #mark #glove = Glove(args.glove_file) #self.glove_embedding = toFloatTensor(glove.glove_embeddings[self.target_object], self.gpu_id) return True
def eval_at_state(self, model_options): model_input = ModelInput() if self.episode.current_frame is None: model_input.state = self.state() else: model_input.state = self.episode.current_frame model_input.hidden = self.hidden current_pos = '{}|{}|{}|{}'.format( # self.environment.scene_name, self.episode.environment.controller.state.position()['x'], self.episode.environment.controller.state.position()['z'], self.episode.environment.controller.state.rotation, self.episode.environment.controller.state.horizon) target_embedding_array = np.zeros((len(CLASSES), 1)) target_embedding_array[CLASSES.index(self.episode.target_object)] = 1 glove_embedding_tensor = np.concatenate( (self.episode.glove_reader[current_pos][()], target_embedding_array), axis=1) # model_input.target_class_embedding = self.episode.glove_embedding # if ((self.episode.glove_reader[current_pos][CLASSES.index(self.episode.target_object)] != np.array([0, 0, 0, 0])).all()) and (self.episode.det_frame == None): # self.episode.current_det = self.eps_len model_input.target_class_embedding = toFloatTensor( glove_embedding_tensor, self.gpu_id) # if self.eps_len == 0: # model_input.det_his = toFloatTensor(torch.zeros(4), self.gpu_id) # else: # model_input.det_his = self.last_det # model_input.det_cur = toFloatTensor(self.episode.glove_reader[current_pos][CLASSES.index(self.episode.target_object)], self.gpu_id) # self.last_det = model_input.det_cur model_input.action_probs = self.last_action_probs # if self.eps_len == 0: # det_iou = toFloatTensor(torch.zeros(1, 4), self.gpu_id) # else: # det_iou = generate_det_4_iou(self.last_det) # det_iou = toFloatTensor(det_iou, self.gpu_id) # # model_input.action_probs = torch.cat((self.last_action_probs, det_iou), dim=1) self.episode.detections.append( self.episode.glove_reader[current_pos][CLASSES.index( self.episode.target_object)]) # self.detector_det = self.episode.glove_reader[current_pos][CLASSES.index(self.episode.target_object)] # if self.episode.task_data[0] in self.episode.det_gt[current_pos]: # self.gt_det = self.episode.det_gt[current_pos][self.episode.task_data[0]] # else: # self.gt_det = np.zeros(4) # optimal_solution = self.episode.environment.controller.shortest_path_to_target(current_pos, self.episode.task_data[0]) if self.episode.optimal_actions is not None: optimal_solution = self.episode.optimal_actions[current_pos] self.optim_step = torch.zeros((1, 6)) self.optim_step[0, optimal_solution] = 1 # if (optimal_solution[0] is not None) and (len(optimal_solution[0]) < 99): # optim_next_state = optimal_solution[0][0] # else: # optim_next_state = current_pos # self.optim_step = self.optim_action(current_pos, optim_next_state) # optim_steps = torch.zeros((1, 1)) # optim_steps[0, 0] = self.episode.environment.controller.shortest_path_to_target(current_pos, self.episode.task_data[0])[1] # model_input.optim_steps = toFloatTensor(optim_steps, self.gpu_id) # det_his = torch.zeros((1, 2)) # if self.episode.current_det: # det_his[0, 1] = 1 # if self.episode.last_det: # det_his[0, 0] = 1 # model_input.det_relation = toFloatTensor(det_his, self.gpu_id) return model_input, self.model.forward(model_input, model_options)
def _new_episode( self, args, scenes, possible_targets, targets=None, keep_obj=False, optimal_act=None, glove=None, det_gt=None, ): """ New navigation episode. """ scene = random.choice(scenes) img_file_scene = args.images_file_name if self._env is None: self._env = Environment( offline_data_dir=args.offline_data_dir, use_offline_controller=True, grid_size=0.25, # images_file_name=args.images_file_name, images_file_name=img_file_scene, local_executable_path=args.local_executable_path, total_images_file=None) self._env.start(scene) else: self._env.reset(scene) # Randomize the start location. self._env.randomize_agent_location() objects = self._env.all_objects() visible_objects = [obj.split("|")[0] for obj in objects] intersection = [obj for obj in visible_objects if obj in targets] self.task_data = [] idx = random.randint(0, len(intersection) - 1) goal_object_type = intersection[idx] self.target_object = goal_object_type for id_ in objects: type_ = id_.split("|")[0] if goal_object_type == type_: self.task_data.append(id_) if args.verbose: print("Scene", scene, "Navigating towards:", goal_object_type) # glove = Glove(os.path.join(args.glove_dir, self.environment.controller.scene_name, 'det_feature.hdf5')) glove = glove[self.environment.controller.scene_name] if optimal_act is not None: self.optimal_actions = optimal_act[ self.environment.controller.scene_name][self.task_data[0]] else: self.optimal_actions = None self.glove_embedding = None init_pos = '{}|{}|{}|{}'.format( # self.environment.controller.scene_name, self.environment.controller.state.position()['x'], self.environment.controller.state.position()['z'], self.environment.controller.state.rotation, self.environment.controller.state.horizon) target_embedding_array = np.zeros((len(CLASSES), 1)) target_embedding_array[CLASSES.index(self.target_object)] = 1 # glove_embedding_tensor = np.concatenate((glove.glove_embeddings[init_pos][()], target_embedding_array), axis=1) glove_embedding_tensor = np.concatenate( (glove[init_pos], target_embedding_array), axis=1) self.glove_embedding = toFloatTensor(glove_embedding_tensor, self.gpu_id) # self.glove_reader = glove.glove_embeddings self.glove_reader = glove
def _new_curriculum_episode(self, args, scenes, possible_targets, targets=None, keep_obj=False, glove=None, protos=None, pre_metadata=None, curriculum_meta=None, total_ep=0): """ New navigation episode. """ # choose a scene scene = None retry = 0 flag_episode_valid = False while not flag_episode_valid: # choose a scene valid_scenes = os.listdir(args.offline_data_dir) intersection_scenes = [ scene for scene in scenes if scene in valid_scenes ] scene = random.choice(intersection_scenes) # TODO: choose difficulty try: diff = round(total_ep // args.num_ep_per_stage) + 1 diff_idx = random.choice(range(diff)) # if total_ep < args.difficulty_upgrade_step: # diff = DIFFICULTY[0] # elif total_ep < 2 * args.difficulty_upgrade_step: # diff = random.choice(DIFFICULTY[:2]) # else: # diff = random.choice(DIFFICULTY[:3]) # choose object # visible_objects = curriculum_meta[scene][diff].keys() # intersection_objs = [obj for obj in visible_objects if obj in targets] # object_type = random.choice(intersection_objs) episode = random.choice(curriculum_meta[scene][diff_idx]) object_type = episode['object_type'].replace(" ", "") if object_type not in targets: continue # to plot trajectory by xiaodong # state_pattern: x, z, rotation_degree, horizon_degree state_pattern = "{:0." + str( args.state_decimal) + "f}|{:0." + str( args.state_decimal) + "f}|{:d}|{:d}" self.init_pos_str = state_pattern.format( episode['initial_position']['x'], episode['initial_position']['z'], episode['initial_orientation'], 0) self.target_pos_str = state_pattern.format( episode['target_position']['x'], episode['target_position']['z'], 0, 0) self.object_type = object_type except: continue # TODO: Present validity checking method breaks the principle of tiered-design and decoupling # TODO: Find a better way to check the validity of an episode by junting, 2020-04-10 state = ThorAgentState(**episode['initial_position'], rotation=episode['initial_orientation'], horizon=0, state_decimal=args.state_decimal) if str(state) in pre_metadata[scene]['all_states']: flag_episode_valid = True else: print( "Episode ID {} not valid for its initial state missing from all_states" .format(episode['id'])) if self._env is None: self._env = Environment( offline_data_dir=args.offline_data_dir, use_offline_controller=True, grid_size=self.grid_size, images_file_name=args.images_file_name, local_executable_path=args.local_executable_path, rotate_by=args.rotate_by, state_decimal=args.state_decimal, pinned_scene=args.pinned_scene, pre_metadata=pre_metadata, actions=self.actions) self._env.start(scene) else: self._env.reset(scene) # initialize the start location. self._env.initialize_agent_location( **episode['initial_position'], rotation=episode['initial_orientation'], horizon=0) self.task_data = [] self.target_object = object_type self.task_data.append(episode['object_id']) self.episode_id = episode['id'] self.episode_trajectories = [] self.actions_taken = [] if args.verbose: print("Episode: Scene ", scene, " Difficulty ", diff, " Navigating towards: ", object_type) if args.glove_file != "": self.glove_embedding = toFloatTensor( glove.glove_embeddings[object_type][:], self.gpu_id) if args.proto_file != "": self.prototype = toFloatTensor( protos.protos[object_type.lower()][:], self.gpu_id) return scene
def _new_random_episode(self, args, scenes, possible_targets, targets=None, keep_obj=False, glove=None, protos=None, pre_metadata=None): """ New navigation episode. """ #random episode scene = None retry = 0 while scene not in os.listdir(args.offline_data_dir): scene = random.choice(scenes) retry += 1 if retry >= 1000: raise Exception("No scenes found in {}".format( args.offline_data_dir)) if self._env is None: self._env = Environment( offline_data_dir=args.offline_data_dir, use_offline_controller=True, grid_size=self.grid_size, images_file_name=args.images_file_name, local_executable_path=args.local_executable_path, rotate_by=args.rotate_by, state_decimal=args.state_decimal, pinned_scene=args.pinned_scene, pre_metadata=pre_metadata, actions=self.actions) self._env.start(scene) else: self._env.reset(scene) # Randomize the start location. self._env.randomize_agent_location() objects = self._env.all_objects() visible_objects = [obj.split("|")[0] for obj in objects] intersection = [obj for obj in visible_objects if obj in targets] self.task_data = [] idx = random.randint(0, len(intersection) - 1) object_type = intersection[idx] self.target_object = object_type for id_ in objects: type_ = id_.split("|")[0] if object_type == type_: self.task_data.append(id_) if args.verbose: print("Scene", scene, "Navigating towards:", object_type) self.episode_trajectories = [] self.actions_taken = [] if args.glove_file != "": self.glove_embedding = toFloatTensor( glove.glove_embeddings[object_type][:], self.gpu_id) if args.proto_file != "": self.prototype = toFloatTensor( protos.protos[object_type.lower()][:], self.gpu_id) return scene