def _new_episode(self,
args,
episode,
glove=None,
protos=None,
pre_metadata=None):
""" New navigation episode. """
if episode == None:
return None
scene = episode["scene"]
if self._env is None:
self._env = Environment(
offline_data_dir=args.offline_data_dir,
use_offline_controller=True,
grid_size=args.grid_size,
images_file_name=args.images_file_name,
local_executable_path=args.local_executable_path,
rotate_by=args.rotate_by,
state_decimal=args.state_decimal,
pinned_scene=args.pinned_scene,
pre_metadata=pre_metadata,
actions=self.actions)
self._env.start(scene)
else:
self._env.reset(scene)
self.environment.controller.state = ThorAgentState(
**episode['initial_position'],
rotation=episode['initial_orientation'],
horizon=0,
state_decimal=args.state_decimal)
self.task_data = [episode["object_id"]]
object_type = episode["object_type"]
self.target_object = object_type
self.difficulty = episode["difficulty"]
self.episode_id = episode['id']
if args.glove_file != "":
self.glove_embedding = toFloatTensor(
glove.glove_embeddings[object_type][:], self.gpu_id)
if args.proto_file != "":
self.prototype = toFloatTensor(
protos.protos[object_type.lower()][:], self.gpu_id)
if args.verbose:
print("Scene", scene, "Navigating towards:", self.target_object)
return scene
def _new_episode(self, args, episode, glove, optimal_act):
""" New navigation episode. """
scene = episode["scene"]
if self._env is None:
self._env = Environment(
offline_data_dir=args.offline_data_dir,
use_offline_controller=True,
grid_size=0.25,
images_file_name=args.images_file_name,
local_executable_path=args.local_executable_path,
)
self._env.start(scene)
else:
self._env.reset(scene)
# self.environment.controller.state = episode["state"]
y = 0.9009995
x, z, hor, rot = episode["state"].split('|')
self.environment.controller.state = ThorAgentState(
float(x), float(y), float(z), float(hor), float(rot))
self.task_data = episode["task_data"]
self.target_object = episode["goal_object_type"]
if args.verbose:
print("Scene", scene, "Navigating towards:", self.target_object)
# self.glove_embedding = gpuify(episode["glove_embedding"], self.gpu_id)
glove = glove[self.environment.controller.scene_name]
self.glove_embedding = None
if optimal_act is not None:
self.optimal_actions = optimal_act[
self.environment.controller.scene_name][self.task_data[0]]
else:
self.optimal_actions = None
init_pos = '{}|{}|{}|{}'.format(
# self.environment.controller.scene_name,
self.environment.controller.state.position()['x'],
self.environment.controller.state.position()['z'],
self.environment.controller.state.rotation,
self.environment.controller.state.horizon)
# init_pos = self.environment.controller.state
target_embedding_array = np.zeros((len(CLASSES), 1))
target_embedding_array[CLASSES.index(self.target_object)] = 1
glove_embedding_tensor = np.concatenate(
(glove[init_pos], target_embedding_array), axis=1)
self.glove_embedding = toFloatTensor(glove_embedding_tensor,
self.gpu_id)
self.glove_reader = glove
return True
def _new_episode(self,
args,
scenes,
possible_targets,
targets=None,
room=None,
keep_obj=False,
glove=None):
""" New navigation episode. """
scene = random.choice(scenes)
self.room = room
if self._env is None:
self._env = Environment(
offline_data_dir=args.offline_data_dir,
use_offline_controller=True,
grid_size=0.25,
images_file_name=args.images_file_name,
local_executable_path=args.local_executable_path,
)
self._env.start(scene)
else:
self._env.reset(scene)
# Randomize the start location.
start_state = self._env.randomize_agent_location()
objects = self._env.all_objects()
visible_objects = [obj.split("|")[0] for obj in objects]
intersection = [obj for obj in visible_objects if obj in targets]
self.task_data = []
idx = random.randint(0, len(intersection) - 1)
goal_object_type = intersection[idx]
self.target_object = goal_object_type
for id_ in objects:
type_ = id_.split("|")[0]
if goal_object_type == type_:
self.task_data.append(id_)
child_object = self.task_data[0].split("|")[0]
#print('room is ', self.room)
try:
self.target_parents = c2p_prob[self.room][child_object]
except:
self.target_parents = None
if args.verbose:
print("Scene", scene, "Navigating towards:", goal_object_type)
self.glove_embedding = None
self.glove_embedding = toFloatTensor(
glove.glove_embeddings[goal_object_type][:], self.gpu_id)
def _new_episode(self, args, episode):
""" New navigation episode. """
scene = episode["scene"]
if "physics" in scene:
scene = scene[:-8]
if self._env is None:
self._env = Environment(
offline_data_dir=args.offline_data_dir,
use_offline_controller=True,
grid_size=0.25,
images_file_name=args.images_file_name,
local_executable_path=args.local_executable_path,
)
self._env.start(scene)
else:
self._env.reset(scene)
self.environment.controller.state = episode["state"]
self.task_data = episode["task_data"]
self.target_object = episode["goal_object_type"]
# with open(metadata_dir + scene + '/visible_object_map.json', 'r')as f :
# meta_data = json.load(f)
# self.task_data = []
# for k, v in meta_data.items():
# obj = k.split('|')[0]
# if self.target_object == obj:
# self.task_data.append(k)
# regex = re.compile('FloorPlan([0-9]*)')
# num = int(regex.findall(scene)[0])
# ind = int(num / 100)
# if ind > 0:
# ind -= 1
# room = rooms[ind]
self.room = episode["room"]
try:
self.target_parents = c2p_prob[self.room][self.target_object]
except KeyError:
print("finding", self.target_object, 'in', self.room)
if args.verbose:
print("Scene", scene, "Navigating towards:", self.target_object)
if args.vis:
log = open('saved_action_' + args.results_json[:-5] + '.log', "a+")
sys.stdout = log
print("Scene", scene, "Navigating towards:", self.target_object)
print(self.environment.controller.state,
self.environment.controller.state.y)
self.glove_embedding = toFloatTensor(episode["glove_embedding"],
self.gpu_id)
#mark
#glove = Glove(args.glove_file)
#self.glove_embedding = toFloatTensor(glove.glove_embeddings[self.target_object], self.gpu_id)
return True
def eval_at_state(self, model_options):
model_input = ModelInput()
if self.episode.current_frame is None:
model_input.state = self.state()
else:
model_input.state = self.episode.current_frame
model_input.hidden = self.hidden
current_pos = '{}|{}|{}|{}'.format(
# self.environment.scene_name,
self.episode.environment.controller.state.position()['x'],
self.episode.environment.controller.state.position()['z'],
self.episode.environment.controller.state.rotation,
self.episode.environment.controller.state.horizon)
target_embedding_array = np.zeros((len(CLASSES), 1))
target_embedding_array[CLASSES.index(self.episode.target_object)] = 1
glove_embedding_tensor = np.concatenate(
(self.episode.glove_reader[current_pos][()],
target_embedding_array),
axis=1)
# model_input.target_class_embedding = self.episode.glove_embedding
# if ((self.episode.glove_reader[current_pos][CLASSES.index(self.episode.target_object)] != np.array([0, 0, 0, 0])).all()) and (self.episode.det_frame == None):
# self.episode.current_det = self.eps_len
model_input.target_class_embedding = toFloatTensor(
glove_embedding_tensor, self.gpu_id)
# if self.eps_len == 0:
# model_input.det_his = toFloatTensor(torch.zeros(4), self.gpu_id)
# else:
# model_input.det_his = self.last_det
# model_input.det_cur = toFloatTensor(self.episode.glove_reader[current_pos][CLASSES.index(self.episode.target_object)], self.gpu_id)
# self.last_det = model_input.det_cur
model_input.action_probs = self.last_action_probs
# if self.eps_len == 0:
# det_iou = toFloatTensor(torch.zeros(1, 4), self.gpu_id)
# else:
# det_iou = generate_det_4_iou(self.last_det)
# det_iou = toFloatTensor(det_iou, self.gpu_id)
#
# model_input.action_probs = torch.cat((self.last_action_probs, det_iou), dim=1)
self.episode.detections.append(
self.episode.glove_reader[current_pos][CLASSES.index(
self.episode.target_object)])
# self.detector_det = self.episode.glove_reader[current_pos][CLASSES.index(self.episode.target_object)]
# if self.episode.task_data[0] in self.episode.det_gt[current_pos]:
# self.gt_det = self.episode.det_gt[current_pos][self.episode.task_data[0]]
# else:
# self.gt_det = np.zeros(4)
# optimal_solution = self.episode.environment.controller.shortest_path_to_target(current_pos, self.episode.task_data[0])
if self.episode.optimal_actions is not None:
optimal_solution = self.episode.optimal_actions[current_pos]
self.optim_step = torch.zeros((1, 6))
self.optim_step[0, optimal_solution] = 1
# if (optimal_solution[0] is not None) and (len(optimal_solution[0]) < 99):
# optim_next_state = optimal_solution[0][0]
# else:
# optim_next_state = current_pos
# self.optim_step = self.optim_action(current_pos, optim_next_state)
# optim_steps = torch.zeros((1, 1))
# optim_steps[0, 0] = self.episode.environment.controller.shortest_path_to_target(current_pos, self.episode.task_data[0])[1]
# model_input.optim_steps = toFloatTensor(optim_steps, self.gpu_id)
# det_his = torch.zeros((1, 2))
# if self.episode.current_det:
# det_his[0, 1] = 1
# if self.episode.last_det:
# det_his[0, 0] = 1
# model_input.det_relation = toFloatTensor(det_his, self.gpu_id)
return model_input, self.model.forward(model_input, model_options)
def _new_episode(
self,
args,
scenes,
possible_targets,
targets=None,
keep_obj=False,
optimal_act=None,
glove=None,
det_gt=None,
):
""" New navigation episode. """
scene = random.choice(scenes)
img_file_scene = args.images_file_name
if self._env is None:
self._env = Environment(
offline_data_dir=args.offline_data_dir,
use_offline_controller=True,
grid_size=0.25,
# images_file_name=args.images_file_name,
images_file_name=img_file_scene,
local_executable_path=args.local_executable_path,
total_images_file=None)
self._env.start(scene)
else:
self._env.reset(scene)
# Randomize the start location.
self._env.randomize_agent_location()
objects = self._env.all_objects()
visible_objects = [obj.split("|")[0] for obj in objects]
intersection = [obj for obj in visible_objects if obj in targets]
self.task_data = []
idx = random.randint(0, len(intersection) - 1)
goal_object_type = intersection[idx]
self.target_object = goal_object_type
for id_ in objects:
type_ = id_.split("|")[0]
if goal_object_type == type_:
self.task_data.append(id_)
if args.verbose:
print("Scene", scene, "Navigating towards:", goal_object_type)
# glove = Glove(os.path.join(args.glove_dir, self.environment.controller.scene_name, 'det_feature.hdf5'))
glove = glove[self.environment.controller.scene_name]
if optimal_act is not None:
self.optimal_actions = optimal_act[
self.environment.controller.scene_name][self.task_data[0]]
else:
self.optimal_actions = None
self.glove_embedding = None
init_pos = '{}|{}|{}|{}'.format(
# self.environment.controller.scene_name,
self.environment.controller.state.position()['x'],
self.environment.controller.state.position()['z'],
self.environment.controller.state.rotation,
self.environment.controller.state.horizon)
target_embedding_array = np.zeros((len(CLASSES), 1))
target_embedding_array[CLASSES.index(self.target_object)] = 1
# glove_embedding_tensor = np.concatenate((glove.glove_embeddings[init_pos][()], target_embedding_array), axis=1)
glove_embedding_tensor = np.concatenate(
(glove[init_pos], target_embedding_array), axis=1)
self.glove_embedding = toFloatTensor(glove_embedding_tensor,
self.gpu_id)
# self.glove_reader = glove.glove_embeddings
self.glove_reader = glove
def _new_curriculum_episode(self,
args,
scenes,
possible_targets,
targets=None,
keep_obj=False,
glove=None,
protos=None,
pre_metadata=None,
curriculum_meta=None,
total_ep=0):
""" New navigation episode. """
# choose a scene
scene = None
retry = 0
flag_episode_valid = False
while not flag_episode_valid:
# choose a scene
valid_scenes = os.listdir(args.offline_data_dir)
intersection_scenes = [
scene for scene in scenes if scene in valid_scenes
]
scene = random.choice(intersection_scenes)
# TODO: choose difficulty
try:
diff = round(total_ep // args.num_ep_per_stage) + 1
diff_idx = random.choice(range(diff))
# if total_ep < args.difficulty_upgrade_step:
# diff = DIFFICULTY[0]
# elif total_ep < 2 * args.difficulty_upgrade_step:
# diff = random.choice(DIFFICULTY[:2])
# else:
# diff = random.choice(DIFFICULTY[:3])
# choose object
# visible_objects = curriculum_meta[scene][diff].keys()
# intersection_objs = [obj for obj in visible_objects if obj in targets]
# object_type = random.choice(intersection_objs)
episode = random.choice(curriculum_meta[scene][diff_idx])
object_type = episode['object_type'].replace(" ", "")
if object_type not in targets:
continue
# to plot trajectory by xiaodong
# state_pattern: x, z, rotation_degree, horizon_degree
state_pattern = "{:0." + str(
args.state_decimal) + "f}|{:0." + str(
args.state_decimal) + "f}|{:d}|{:d}"
self.init_pos_str = state_pattern.format(
episode['initial_position']['x'],
episode['initial_position']['z'],
episode['initial_orientation'], 0)
self.target_pos_str = state_pattern.format(
episode['target_position']['x'],
episode['target_position']['z'], 0, 0)
self.object_type = object_type
except:
continue
# TODO: Present validity checking method breaks the principle of tiered-design and decoupling
# TODO: Find a better way to check the validity of an episode by junting, 2020-04-10
state = ThorAgentState(**episode['initial_position'],
rotation=episode['initial_orientation'],
horizon=0,
state_decimal=args.state_decimal)
if str(state) in pre_metadata[scene]['all_states']:
flag_episode_valid = True
else:
print(
"Episode ID {} not valid for its initial state missing from all_states"
.format(episode['id']))
if self._env is None:
self._env = Environment(
offline_data_dir=args.offline_data_dir,
use_offline_controller=True,
grid_size=self.grid_size,
images_file_name=args.images_file_name,
local_executable_path=args.local_executable_path,
rotate_by=args.rotate_by,
state_decimal=args.state_decimal,
pinned_scene=args.pinned_scene,
pre_metadata=pre_metadata,
actions=self.actions)
self._env.start(scene)
else:
self._env.reset(scene)
# initialize the start location.
self._env.initialize_agent_location(
**episode['initial_position'],
rotation=episode['initial_orientation'],
horizon=0)
self.task_data = []
self.target_object = object_type
self.task_data.append(episode['object_id'])
self.episode_id = episode['id']
self.episode_trajectories = []
self.actions_taken = []
if args.verbose:
print("Episode: Scene ", scene, " Difficulty ", diff,
" Navigating towards: ", object_type)
if args.glove_file != "":
self.glove_embedding = toFloatTensor(
glove.glove_embeddings[object_type][:], self.gpu_id)
if args.proto_file != "":
self.prototype = toFloatTensor(
protos.protos[object_type.lower()][:], self.gpu_id)
return scene
def _new_random_episode(self,
args,
scenes,
possible_targets,
targets=None,
keep_obj=False,
glove=None,
protos=None,
pre_metadata=None):
""" New navigation episode. """
#random episode
scene = None
retry = 0
while scene not in os.listdir(args.offline_data_dir):
scene = random.choice(scenes)
retry += 1
if retry >= 1000:
raise Exception("No scenes found in {}".format(
args.offline_data_dir))
if self._env is None:
self._env = Environment(
offline_data_dir=args.offline_data_dir,
use_offline_controller=True,
grid_size=self.grid_size,
images_file_name=args.images_file_name,
local_executable_path=args.local_executable_path,
rotate_by=args.rotate_by,
state_decimal=args.state_decimal,
pinned_scene=args.pinned_scene,
pre_metadata=pre_metadata,
actions=self.actions)
self._env.start(scene)
else:
self._env.reset(scene)
# Randomize the start location.
self._env.randomize_agent_location()
objects = self._env.all_objects()
visible_objects = [obj.split("|")[0] for obj in objects]
intersection = [obj for obj in visible_objects if obj in targets]
self.task_data = []
idx = random.randint(0, len(intersection) - 1)
object_type = intersection[idx]
self.target_object = object_type
for id_ in objects:
type_ = id_.split("|")[0]
if object_type == type_:
self.task_data.append(id_)
if args.verbose:
print("Scene", scene, "Navigating towards:", object_type)
self.episode_trajectories = []
self.actions_taken = []
if args.glove_file != "":
self.glove_embedding = toFloatTensor(
glove.glove_embeddings[object_type][:], self.gpu_id)
if args.proto_file != "":
self.prototype = toFloatTensor(
protos.protos[object_type.lower()][:], self.gpu_id)
return scene