def get_3d_location(self, exploration_image, data_point, panaroma=True):
state = AgentObservedState(
instruction=data_point.instruction,
config=self.config,
constants=self.constants,
start_image=exploration_image,
previous_action=None,
pose=None,
position_orientation=data_point.get_start_pos(),
data_point=data_point)
volatile = self.predictor_model.get_attention_prob(state,
model_state=None)
inferred_ix = int(
torch.max(volatile["attention_logits"],
0)[1].data.cpu().numpy()[0])
# Max pointed about that when inferred ix above is the last value then calculations are buggy. He is right.
predicted_row = int(inferred_ix / float(192))
predicted_col = inferred_ix % 192
screen_pos = (predicted_row, predicted_col)
if panaroma:
# Index of the 6 image where the goal is
region_index = int(predicted_col / 32)
predicted_col = predicted_col % 32 # Column within that image where the goal is
pos = data_point.get_start_pos()
new_pos_angle = GoalPredictionSingle360ImageSupervisedLearningFromDisk.\
get_new_pos_angle_from_region_index(region_index, pos)
metadata = {
"x_pos": pos[0],
"z_pos": pos[1],
"y_angle": new_pos_angle
}
else:
pos = data_point.get_start_pos()
metadata = {"x_pos": pos[0], "z_pos": pos[1], "y_angle": pos[2]}
row, col = predicted_row + 0.5, predicted_col + 0.5
start_pos = current_pos_from_metadata(metadata)
start_pose = current_pose_from_metadata(metadata)
goal_pos = data_point.get_destination_list()[-1]
height_drone = 2.5
x_gen, z_gen = get_inverse_object_position(
row, col, height_drone, 30, 32, 32,
(start_pos[0], start_pos[1], start_pose))
predicted_goal_pos = (x_gen, z_gen)
x_goal, z_goal = goal_pos
x_diff = x_gen - x_goal
z_diff = z_gen - z_goal
dist = math.sqrt(x_diff * x_diff + z_diff * z_diff)
return predicted_goal_pos, dist, screen_pos, volatile[
"attention_probs"]
def parse(folder_name, dataset, model):
start = time.time()
num_channel, height, width = model.image_module.get_final_dimension()
# Read images
image_dataset = []
num_examples = len(os.listdir(folder_name))
for i in range(0, num_examples):
example_folder_name = folder_name + "/example_" + str(i)
img = scipy.misc.imread(example_folder_name +
"/image_0.png").swapaxes(1,
2).swapaxes(0, 1)
images = [img]
image_dataset.append(images)
assert len(image_dataset) == len(dataset)
# Read the goal state. The data for the single image can be
# directly computed and does not need to be saved.
goal_dataset = []
for i in range(0, num_examples):
data_point = dataset[i]
pos = data_point.get_start_pos()
metadata = {"x_pos": pos[0], "z_pos": pos[1], "y_angle": pos[2]}
goal_location = [
GoalPrediction.get_goal_location(metadata, data_point, height,
width)
]
_, _, row, col = goal_location[0]
start_pos = current_pos_from_metadata(metadata)
start_pose = current_pose_from_metadata(metadata)
if row is not None and col is not None:
goal_pos = data_point.get_destination_list()[-1]
x_goal, z_goal = goal_pos
height_drone = 2.5
x_gen, z_gen = get_inverse_object_position(
row, col, height_drone, 30, height, width,
(start_pos[0], start_pos[1], start_pose))
x_diff = x_gen - x_goal
z_diff = z_gen - z_goal
dist = math.sqrt(x_diff * x_diff + z_diff * z_diff)
assert dist < 0.5, "forward computation of goal should match inverse computation"
goal_dataset.append(goal_location)
end = time.time()
logging.info("Parsed dataset of size %r in time % seconds",
len(image_dataset), (end - start))
return image_dataset, goal_dataset
def compute_distance_in_real_world(self, inferred_ix, row_col, data_point, panaroma=True):
if row_col is None:
predicted_row = int(inferred_ix / float(self.final_width))
predicted_col = inferred_ix % self.final_width
else:
predicted_row, predicted_col = row_col
if panaroma:
region_index = int(predicted_col / 32)
predicted_col = predicted_col % 32
pos = data_point.get_start_pos()
new_pos_angle = GoalPredictionSingle360ImageSupervisedLearningFromDisk.\
get_new_pos_angle_from_region_index(region_index, pos)
metadata = {"x_pos": pos[0], "z_pos": pos[1], "y_angle": new_pos_angle}
else:
pos = data_point.get_start_pos()
metadata = {"x_pos": pos[0], "z_pos": pos[1], "y_angle": pos[2]}
if row_col is None:
row, col = predicted_row + 0.5, predicted_col + 0.5
else:
row, col = predicted_row, predicted_col
start_pos = current_pos_from_metadata(metadata)
start_pose = current_pose_from_metadata(metadata)
goal_pos = data_point.get_destination_list()[-1]
height_drone = 2.5
x_gen, z_gen = get_inverse_object_position(row, col, height_drone, 30, 32, 32,
(start_pos[0], start_pos[1], start_pose))
x_goal, z_goal = goal_pos
x_diff = x_gen - x_goal
z_diff = z_gen - z_goal
dist = math.sqrt(x_diff * x_diff + z_diff * z_diff)
return (x_gen, z_gen), dist
def compute_distance_in_real_world(self, inferred_ix, data_point):
predicted_row = int(inferred_ix / float(self.final_width))
predicted_col = inferred_ix % self.final_width
row, col = predicted_row + 0.5, predicted_col + 0.5
pos = data_point.get_start_pos()
metadata = {"x_pos": pos[0], "z_pos": pos[1], "y_angle": pos[2]}
start_pos = current_pos_from_metadata(metadata)
start_pose = current_pose_from_metadata(metadata)
goal_pos = data_point.get_destination_list()[-1]
height_drone = 2.5
x_gen, z_gen = get_inverse_object_position(
row, col, height_drone, 30, self.final_height, self.final_width,
(start_pos[0], start_pos[1], start_pose))
x_goal, z_goal = goal_pos
x_diff = x_gen - x_goal
z_diff = z_gen - z_goal
dist = math.sqrt(x_diff * x_diff + z_diff * z_diff)
return dist
def get_3d_location_for_paragraphs(self,
exploration_image,
instruction,
start_pos,
goal_pos,
panaroma=True):
state = AgentObservedState(instruction=instruction,
config=self.config,
constants=self.constants,
start_image=exploration_image,
previous_action=None,
pose=None,
position_orientation=start_pos,
data_point=None)
volatile = self.predictor_model.get_attention_prob(state,
model_state=None)
inferred_ix = int(
torch.max(volatile["attention_logits"],
0)[1].data.cpu().numpy()[0])
########################################
# inst_string = instruction_to_string(instruction, self.config)
# self.save_attention_prob(exploration_image, volatile["attention_probs"][:-1].view(32, 192), inst_string)
########################################
predicted_row = int(inferred_ix / float(192))
predicted_col = inferred_ix % 192
if panaroma:
# Index of the 6 image where the goal is
region_index = int(predicted_col / 32)
predicted_col = predicted_col % 32 # Column within that image where the goal is
pos = start_pos
new_pos_angle = GoalPredictionSingle360ImageSupervisedLearningFromDisk.\
get_new_pos_angle_from_region_index(region_index, pos)
metadata = {
"x_pos": pos[0],
"z_pos": pos[1],
"y_angle": new_pos_angle
}
else:
pos = start_pos
metadata = {"x_pos": pos[0], "z_pos": pos[1], "y_angle": pos[2]}
row, col = predicted_row + 0.5, predicted_col + 0.5
start_pos = current_pos_from_metadata(metadata)
start_pose = current_pose_from_metadata(metadata)
height_drone = 2.5
x_gen, z_gen = get_inverse_object_position(
row, col, height_drone, 30, 32, 32,
(start_pos[0], start_pos[1], start_pose))
predicted_goal_pos = (x_gen, z_gen)
x_goal, z_goal = goal_pos
x_diff = x_gen - x_goal
z_diff = z_gen - z_goal
dist = math.sqrt(x_diff * x_diff + z_diff * z_diff)
return predicted_goal_pos, dist
def parse(folder_name, dataset):
start = time.time()
image_dataset = []
num_examples = len(os.listdir(folder_name))
for i in range(0, num_examples):
example_folder_name = folder_name + "/example_" + str(i)
image_names = [
file for file in os.listdir(example_folder_name)
if file.endswith('.png')
]
num_actions = len(image_names)
images = []
for j in range(0, num_actions):
img = scipy.misc.imread(example_folder_name +
"/image_" + str(j) + ".png").swapaxes(
1, 2).swapaxes(0, 1)
images.append(img)
image_dataset.append(images)
# goal_dataset = []
# num_examples = len(os.listdir(folder_name))
# for i in range(0, num_examples):
# example_folder_name = folder_name + "/example_" + str(i)
# lines = open(example_folder_name + "/goal.txt").readlines()
# goals = []
# for line in lines:
# words = line.strip().split()
# assert len(words) == 4
# if words[0] == "None" or words[1] == "None":
# row, col, row_real, col_real = None, None, None, None
# else:
# row, col, row_real, col_real = int(words[0]), int(words[1]), float(words[2]), float(words[3])
# assert 0 <= row < 8 and 0 <= col < 8
# goals.append((row, col, row_real, col_real))
# goal_dataset.append(goals)
#
# assert len(image_dataset) == len(goal_dataset)
assert len(image_dataset) == len(dataset)
####################################
# Hack for synythetic data
goal_dataset = []
for i in range(0, num_examples):
data_point = dataset[i]
pos = data_point.get_start_pos()
metadata = {"x_pos": pos[0], "z_pos": pos[1], "y_angle": pos[2]}
goal_location = [
GoalPrediction.get_goal_location(metadata, data_point, 32, 32)
]
_, _, row, col = goal_location[0]
start_pos = current_pos_from_metadata(metadata)
start_pose = current_pose_from_metadata(metadata)
if row is not None and col is not None:
goal_pos = data_point.get_destination_list()[-1]
height_drone = 2.5
x_gen, y_gen = get_inverse_object_position(
row, col, height_drone, 30, 32, 32,
(start_pos[0], start_pos[1], start_pose))
goal_dataset.append(goal_location)
data_point.trajectory = [0]
# if len(image_dataset[i]) >= 2:
# data_point.trajectory = [0] # dummy action added
#####################################
end = time.time()
logging.info("Parsed dataset of size %r in time % seconds",
len(image_dataset), (end - start))
return image_dataset, goal_dataset
def parse_oracle_turn(folder_name, dataset, model):
start = time.time()
num_channel, height, width = model.image_module.get_final_dimension()
# Read images
image_dataset = []
num_examples = len(os.listdir(folder_name))
for i in range(0, num_examples):
data_point = dataset[i]
################################################
pos = data_point.get_start_pos()
metadata = {"x_pos": pos[0], "z_pos": pos[1], "y_angle": pos[2]}
turn_angle = get_turn_angle_from_metadata_datapoint(metadata, data_point)
assert 180.0 >= turn_angle >= -180.0
if 30.0 >= turn_angle > -30.0:
ix = 0
mean_turn_angle = 0
elif 90.0 >= turn_angle > 30.0:
ix = 1
mean_turn_angle = 60
elif 150.0 >= turn_angle > 90.0:
ix = 2
mean_turn_angle = 120
elif -30 >= turn_angle > -90.0:
ix = 5
mean_turn_angle = -60
elif -90.0 >= turn_angle > -150.0:
ix = 4
mean_turn_angle = -120
else:
ix = 3
mean_turn_angle = 180
print("Pose is %r, Turn Angle is %r and Mean Turn Angle is %r " % (pos[2], turn_angle, mean_turn_angle))
new_pos_angle = pos[2] + mean_turn_angle
while new_pos_angle < -180:
new_pos_angle += 360.0
while new_pos_angle > 180:
new_pos_angle -= 360.0
# Modify the pos to turn towards the image
new_pos = (pos[0], pos[1], new_pos_angle)
data_point.start_pos = new_pos
pos = data_point.get_start_pos()
metadata = {"x_pos": pos[0], "z_pos": pos[1], "y_angle": pos[2]}
new_turn_angle = get_turn_angle_from_metadata_datapoint(metadata, data_point)
assert 30.0 >= new_turn_angle >= -30.0, "Found turn angle of " + str(new_turn_angle)
################################################
example_folder_name = folder_name + "/example_" + str(i)
img = scipy.misc.imread(example_folder_name + "/image_" + str(ix) + ".png").swapaxes(1, 2).swapaxes(0, 1)
images = [img]
image_dataset.append(images)
assert len(image_dataset) == len(dataset)
# Read the goal state. The data for the single image can be
# directly computed and does not need to be saved.
goal_dataset = []
for i in range(0, num_examples):
data_point = dataset[i]
pos = data_point.get_start_pos()
metadata = {"x_pos": pos[0], "z_pos": pos[1], "y_angle": pos[2]}
goal_location = [GoalPrediction.get_goal_location(metadata, data_point, height, width)]
_, _, row, col = goal_location[0]
start_pos = current_pos_from_metadata(metadata)
start_pose = current_pose_from_metadata(metadata)
if row is not None and col is not None:
goal_pos = data_point.get_destination_list()[-1]
x_goal, z_goal = goal_pos
height_drone = 2.5
x_gen, z_gen = get_inverse_object_position(row, col, height_drone, 30, height, width,
(start_pos[0], start_pos[1], start_pose))
x_diff = x_gen - x_goal
z_diff = z_gen - z_goal
dist = math.sqrt(x_diff * x_diff + z_diff * z_diff)
assert dist < 0.5, "forward computation of goal should match inverse computation"
else:
print("Warning: None found! ")
goal_dataset.append(goal_location)
end = time.time()
logging.info("Parsed dataset of size %r in time % seconds", len(image_dataset), (end - start))
return image_dataset, goal_dataset