def eval_at_state(self, model_options):
model_input = ModelInput()
if self.episode.current_frame is None:
model_input.state = self.state()
else:
model_input.state = self.episode.current_frame
if self.episode.current_objs is None:
model_input.objbb = self.objstate()
else:
model_input.objbb = self.episode.current_objs
model_input.hidden = self.hidden
model_input.target_class_embedding = self.episode.glove_embedding
model_input.action_probs = self.last_action_probs
return model_input, self.model.forward(model_input, model_options)
def eval_at_state(self, model_options):
model_input = ModelInput()
if self.episode.current_frame is None:
model_input.state = self.state(
) # [1,512,7,7] feature of current state from ResNet50
else:
model_input.state = self.episode.current_frame
model_input.hidden = self.hidden
# choose target_class_embedding
if self.model_name == "ProtoModel":
model_input.target_class_embedding = self.episode.prototype # [1, 512, 7, 7]
else:
model_input.target_class_embedding = self.episode.glove_embedding # [300]
model_input.action_probs = self.last_action_probs # [1, #(ACTION_SPACE)]
return model_input, self.model.forward(model_input, model_options)
def eval_at_state(self, model_options, frame):
model_input = ModelInput()
# if self.episode.current_frame is None:
# model_input.state = self.state()
# else:
# model_input.state = self.episode.current_frame
#process_frame to shape [1,3,224,224], for input to resnet18
processed_frame = self.preprocess_frame(
resnet_input_transform(frame, 224).unsqueeze(0))
resnet18_features = self.resnet18(processed_frame)
model_input.state = resnet18_features
model_input.hidden = self.hidden
model_input.target_class_embedding = gpuify(torch.Tensor(
self.target_glove_embedding),
gpu_id=self.gpu_id)
model_input.action_probs = self.last_action_probs
return model_input, self.model.forward(model_input, model_options)
def eval_at_state(self, model_options):
model_input = ModelInput()
if self.episode.current_frame is None:
model_input.state = self.state()
else:
model_input.state = self.episode.current_frame
model_input.hidden = self.hidden
current_pos = '{}|{}|{}|{}'.format(
# self.environment.scene_name,
self.episode.environment.controller.state.position()['x'],
self.episode.environment.controller.state.position()['z'],
self.episode.environment.controller.state.rotation,
self.episode.environment.controller.state.horizon)
target_embedding_array = np.zeros((len(CLASSES), 1))
target_embedding_array[CLASSES.index(self.episode.target_object)] = 1
glove_embedding_tensor = np.concatenate(
(self.episode.glove_reader[current_pos][()],
target_embedding_array),
axis=1)
# model_input.target_class_embedding = self.episode.glove_embedding
# if ((self.episode.glove_reader[current_pos][CLASSES.index(self.episode.target_object)] != np.array([0, 0, 0, 0])).all()) and (self.episode.det_frame == None):
# self.episode.current_det = self.eps_len
model_input.target_class_embedding = toFloatTensor(
glove_embedding_tensor, self.gpu_id)
# if self.eps_len == 0:
# model_input.det_his = toFloatTensor(torch.zeros(4), self.gpu_id)
# else:
# model_input.det_his = self.last_det
# model_input.det_cur = toFloatTensor(self.episode.glove_reader[current_pos][CLASSES.index(self.episode.target_object)], self.gpu_id)
# self.last_det = model_input.det_cur
model_input.action_probs = self.last_action_probs
# if self.eps_len == 0:
# det_iou = toFloatTensor(torch.zeros(1, 4), self.gpu_id)
# else:
# det_iou = generate_det_4_iou(self.last_det)
# det_iou = toFloatTensor(det_iou, self.gpu_id)
#
# model_input.action_probs = torch.cat((self.last_action_probs, det_iou), dim=1)
self.episode.detections.append(
self.episode.glove_reader[current_pos][CLASSES.index(
self.episode.target_object)])
# self.detector_det = self.episode.glove_reader[current_pos][CLASSES.index(self.episode.target_object)]
# if self.episode.task_data[0] in self.episode.det_gt[current_pos]:
# self.gt_det = self.episode.det_gt[current_pos][self.episode.task_data[0]]
# else:
# self.gt_det = np.zeros(4)
# optimal_solution = self.episode.environment.controller.shortest_path_to_target(current_pos, self.episode.task_data[0])
if self.episode.optimal_actions is not None:
optimal_solution = self.episode.optimal_actions[current_pos]
self.optim_step = torch.zeros((1, 6))
self.optim_step[0, optimal_solution] = 1
# if (optimal_solution[0] is not None) and (len(optimal_solution[0]) < 99):
# optim_next_state = optimal_solution[0][0]
# else:
# optim_next_state = current_pos
# self.optim_step = self.optim_action(current_pos, optim_next_state)
# optim_steps = torch.zeros((1, 1))
# optim_steps[0, 0] = self.episode.environment.controller.shortest_path_to_target(current_pos, self.episode.task_data[0])[1]
# model_input.optim_steps = toFloatTensor(optim_steps, self.gpu_id)
# det_his = torch.zeros((1, 2))
# if self.episode.current_det:
# det_his[0, 1] = 1
# if self.episode.last_det:
# det_his[0, 0] = 1
# model_input.det_relation = toFloatTensor(det_his, self.gpu_id)
return model_input, self.model.forward(model_input, model_options)
def eval_at_state(self, model_options):
model_input = ModelInput()
# model inputs
if self.episode.current_frame is None:
model_input.state = self.state()
else:
model_input.state = self.episode.current_frame
model_input.hidden = self.hidden
current_detection_feature = self.episode.current_detection_feature()
current_detection_feature = current_detection_feature[
self.targets_index, :]
target_embedding_array = np.zeros((len(self.targets), 1))
target_embedding_array[self.targets.index(
self.episode.target_object)] = 1
self.episode.detection_results.append(
list(current_detection_feature[
self.targets.index(self.episode.target_object), 512:]))
target_embedding = {
'appear': current_detection_feature[:, :512],
'info': current_detection_feature[:, 512:],
'indicator': target_embedding_array
}
target_embedding['appear'] = toFloatTensor(target_embedding['appear'],
self.gpu_id)
target_embedding['info'] = toFloatTensor(target_embedding['info'],
self.gpu_id)
target_embedding['indicator'] = toFloatTensor(
target_embedding['indicator'], self.gpu_id)
model_input.target_class_embedding = target_embedding
model_input.action_probs = self.last_action_probs
if 'Memory' in self.model_name:
state_length = self.hidden_state_sz
if len(self.episode.state_reps) == 0:
model_input.states_rep = torch.zeros(1, state_length)
else:
model_input.states_rep = torch.stack(self.episode.state_reps)
dim_obs = 512
if len(self.episode.obs_reps) == 0:
model_input.obs_reps = torch.zeros(1, dim_obs)
else:
model_input.obs_reps = torch.stack(self.episode.obs_reps)
if len(self.episode.state_memory) == 0:
model_input.states_memory = torch.zeros(1, state_length)
else:
model_input.states_memory = torch.stack(
self.episode.state_memory)
if len(self.episode.action_memory) == 0:
model_input.action_memory = torch.zeros(1, 6)
else:
model_input.action_memory = torch.stack(
self.episode.action_memory)
model_input.states_rep = toFloatTensor(model_input.states_rep,
self.gpu_id)
model_input.states_memory = toFloatTensor(
model_input.states_memory, self.gpu_id)
model_input.action_memory = toFloatTensor(
model_input.action_memory, self.gpu_id)
model_input.obs_reps = toFloatTensor(model_input.obs_reps,
self.gpu_id)
return model_input, self.model.forward(model_input, model_options)