def __str__(self):
""" Get the string representation of the question """
if self.parent_object_class is None:
return game_util.get_question_str(
0, OBJECT_CLASS_TO_ID[self.object_class])
else:
return game_util.get_question_str(
2, OBJECT_CLASS_TO_ID[self.object_class],
OBJECT_CLASS_TO_ID[self.parent_object_class])
def reset(self, seed=None, test_ind=None):
self.board = None
self.seen_object = False
self.terminal = False
self.opened_receptacles = set()
self.closed_receptacles = set()
self.seen_obj1 = set()
self.seen_obj2 = set()
self.visited_locations = set()
self.can_end = False
if seed is not None:
self.local_random.seed(seed)
# Do equal number of each question type in train.
question_type_ind = self.local_random.sample(
constants.USED_QUESTION_TYPES, 1)[0]
# get random row
if test_ind is not None:
question_row, question_type_ind = test_ind
question_type = self.question_types[question_type_ind]
question_data = self.test_datasets[question_type_ind][
question_row, :]
test_ind = (question_row, question_type_ind)
else:
question_type = self.question_types[question_type_ind]
question_row = self.local_random.randint(
0,
len(self.datasets[question_type_ind]) - 1)
question_data = self.datasets[question_type_ind][question_row, :]
container_ind = None
if question_type_ind == 0 or question_type_ind == 1:
scene_num, scene_seed, object_ind, answer = question_data
self.question_target = object_ind
if question_type_ind == 0:
answer = bool(answer)
elif question_type_ind == 2:
scene_num, scene_seed, object_ind, container_ind, answer = question_data
answer = bool(answer)
self.question_target = (object_ind, container_ind)
else:
raise Exception('No question type found for type %d' %
question_type_ind)
self.scene_seed = scene_seed
self.scene_num = scene_num
self.object_target = object_ind
self.parent_target = container_ind
self.container_target = np.zeros(constants.NUM_CLASSES)
self.direction_target = np.zeros(4)
if container_ind is not None:
self.container_target[container_ind] = 1
self.question_type_ind = question_type_ind
self.scene_name = 'FloorPlan%d' % scene_num
grid_file = 'layouts/%s-layout.npy' % self.scene_name
self.graph = graph_obj.Graph(grid_file, use_gt=False)
self.xray_graph = graph_obj.Graph(grid_file, use_gt=True)
self.bounds = [
self.graph.xMin, self.graph.yMin,
self.graph.xMax - self.graph.xMin + 1,
self.graph.yMax - self.graph.yMin + 1
]
max_num_repeats = 1
remove_prob = 0.5
if question_type == 'existence':
max_num_repeats = 10
remove_prob = 0.25
elif question_type == 'counting':
max_num_repeats = constants.MAX_COUNTING_ANSWER + 1
remove_prob = 0.5
elif question_type == 'contains':
max_num_repeats = 10
remove_prob = 0.25
self.event = game_util.reset(self.env, self.scene_name)
self.agent_height = self.event.metadata['agent']['position']['y']
self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET
self.event = self.env.random_initialize(
self.scene_seed,
max_num_repeats=max_num_repeats,
remove_prob=remove_prob)
print('Type:', question_type, 'Row: ', question_row, 'Scene',
self.scene_name, 'seed', scene_seed)
print(
'Question:',
game_util.get_question_str(question_type_ind, object_ind,
container_ind))
if self.question_type_ind == 2:
print('Answer:', constants.OBJECTS[object_ind], 'in',
constants.OBJECTS[container_ind], 'is', answer)
else:
print('Answer:', constants.OBJECTS[object_ind], 'is', answer)
self.answer = answer
# Verify answer
if self.question_type_ind == 0:
objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind],
self.event.metadata)
computed_answer = len(objs) > 0
requires_interaction = True
for obj in objs:
parent = obj['parentReceptacle'].split('|')[0]
if parent not in {'Fridge', 'Cabinet', 'Microwave'}:
requires_interaction = False
break
elif self.question_type_ind == 1:
objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind],
self.event.metadata)
computed_answer = len(objs)
requires_interaction = True
elif self.question_type_ind == 2:
objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind],
self.event.metadata)
if len(objs) == 0:
computed_answer = False
requires_interaction = constants.OBJECTS[
self.question_target[1]] in {
'Fridge', 'Cabinet', 'Microwave'
}
else:
obj = objs[0]
computed_answer = False
for obj in objs:
requires_interaction = True
parent = obj['parentReceptacle'].split('|')[0]
if parent in constants.OBJECT_CLASS_TO_ID:
parent_ind = constants.OBJECT_CLASS_TO_ID[parent]
computed_answer = parent_ind == self.question_target[1]
if computed_answer:
if parent not in {
'Fridge', 'Cabinet', 'Microwave'
}:
requires_interaction = False
break
else:
computed_answer = False
self.requires_interaction = requires_interaction
try:
assert self.answer == computed_answer, 'Answer does not match scene metadata'
except AssertionError:
print('Type:', question_type, 'Row: ', question_row, 'Scene',
self.scene_name, 'seed', scene_seed)
print('Answer', computed_answer, 'does not match expected value',
self.answer, ', did randomization process change?')
pdb.set_trace()
self.answer = computed_answer
if constants.NUM_CLASSES > 1:
self.hidden_items = set()
objects = self.event.metadata['objects']
for obj in objects:
if obj['receptacle'] and obj['openable'] and not obj['isopen']:
for inside_obj in obj['receptacleObjectIds']:
self.hidden_items.add(inside_obj)
objects = self.event.metadata['objects']
for obj in objects:
if obj['objectType'] not in constants.OBJECT_CLASS_TO_ID:
continue
obj_bounds = game_util.get_object_bounds(obj, self.bounds)
self.xray_graph.memory[
obj_bounds[1]:obj_bounds[3], obj_bounds[0]:obj_bounds[2],
constants.OBJECT_CLASS_TO_ID[obj['objectType']] + 1] = 1
start_point = self.local_random.randint(0,
self.graph.points.shape[0] - 1)
start_point = self.graph.points[start_point, :].copy()
self.start_point = (start_point[0], start_point[1],
self.local_random.randint(0, 3))
action = {
'action': 'TeleportFull',
'x': self.start_point[0] * constants.AGENT_STEP_SIZE,
'y': self.agent_height,
'z': self.start_point[1] * constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'rotation': self.start_point[2] * 90,
'horizon': 30,
}
self.event = self.env.step(action)
self.process_frame()
self.reward = 0
self.end_point = []