def step(self, action_or_ind):
self.reward = -0.01
action, teleport_failure, should_fail = self.get_action(action_or_ind)
t_start = time.time()
if should_fail or teleport_failure:
self.event.metadata['lastActionSuccess'] = False
else:
if action[
'action'] != 'Teleport' or not constants.USE_NAVIGATION_AGENT:
self.event = self.env.step(action)
else:
# Action is teleport and I should do low level navigation.
pass
new_pose = game_util.get_pose(self.event)
point_dists = np.sum(np.abs(self.graph.points -
np.array(new_pose)[:2]),
axis=1)
if np.min(point_dists) > 0.0001:
print('Point teleport failure')
closest_point = self.graph.points[np.argmin(point_dists)]
self.event = self.env.step({
'action': 'Teleport',
'x': closest_point[0] * constants.AGENT_STEP_SIZE,
'y': self.agent_height,
'z': closest_point[1] * constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'rotation': self.pose[2] * 90,
})
else:
closest_point = np.argmin(point_dists)
if closest_point not in self.visited_locations:
self.visited_locations.add(closest_point)
self.times[2, 0] += time.time() - t_start
self.times[2, 1] += 1
if self.times[2, 1] % 100 == 0:
print('env step time %.3f' % (self.times[2, 0] / self.times[2, 1]))
if self.event.metadata['lastActionSuccess']:
self.process_frame()
if action['action'] == 'OpenObject':
if self.question_type_ind == 2 and action['objectId'].split(
'|')[0] != constants.OBJECTS[self.question_target[1]]:
self.reward -= 1.0
elif action['objectId'] not in self.opened_receptacles:
if self.question_type_ind == 2 and action[
'objectId'].split('|')[0] == constants.OBJECTS[
self.question_target[1]]:
self.reward += 5.0
else:
self.reward += 0.1
self.opened_receptacles.add(action['objectId'])
elif action[
'action'] == 'CloseObject' and self.question_type_ind != 2:
if action['objectId'] not in self.closed_receptacles:
self.reward += 0.1
self.closed_receptacles.add(action['objectId'])
# Update seen objects related to question
objs = game_util.get_objects_of_type(
constants.OBJECTS[self.object_target], self.event.metadata)
objs = [
obj for obj in objs
if (obj['objectId'] in self.event.instance_detections2D
and game_util.check_object_size(
self.event.instance_detections2D[obj['objectId']]))
]
for obj in objs:
self.seen_obj1.add(obj['objectId'])
if self.question_type_ind in {2, 3}:
objs = game_util.get_objects_of_type(
constants.OBJECTS[self.question_target[1]],
self.event.metadata)
objs = [
obj for obj in objs
if (obj['objectId'] in self.event.instance_detections2D
and game_util.check_object_size(
self.event.instance_detections2D[obj['objectId']]))
]
for obj in objs:
self.seen_obj2.add(obj['objectId'])
if not self.can_end:
if self.question_type_ind == 0:
self.can_end = len(self.seen_obj1) > 0
elif self.question_type_ind == 1:
self.can_end = len(self.seen_obj1) == self.answer
elif self.question_type_ind == 2:
objs = game_util.get_objects_of_type(
constants.OBJECTS[self.question_target[1]],
self.event.metadata)
if not self.answer:
if objs[0]['openable']:
if all([
obj['objectId'] in self.opened_receptacles
for obj in objs
]):
self.can_end = True
else:
if all([
obj['objectId'] in self.seen_obj2
for obj in objs
]):
self.can_end = True
else:
objs = [
obj for obj in objs
if (obj['objectId'] in
self.event.instance_detections2D
and game_util.check_object_size(
self.event.instance_detections2D[
obj['objectId']]))
]
for obj in objs:
for contained_obj in obj['pivotSimObjs']:
if contained_obj['objectId'] in self.seen_obj1:
self.can_end = True
else:
self.reward -= 0.05
def create_dump():
time_str = py_util.get_time_str()
prefix = 'questions/'
if not os.path.exists(prefix + dataset_type + '/data_contains'):
os.makedirs(prefix + dataset_type + '/data_contains')
h5 = h5py.File(prefix + dataset_type + '/data_contains/Contains_Questions_' + time_str + '.h5', 'w')
h5.create_dataset('questions/question', (num_record, 5), dtype=np.int32)
print('Generating %d contains questions' % num_record)
# Generate contains questions
data_ind = 0
episode = Episode()
scene_number = -1
while data_ind < num_record:
k = 0
scene_number += 1
scene_num = scene_numbers[scene_number % len(scene_numbers)]
scene_name = 'FloorPlan%d' % scene_num
episode.initialize_scene(scene_name)
num_tries = 0
while k < num_samples_per_scene and num_tries < 10 * num_samples_per_scene:
# randomly pick a pickable object in the scene
object_class = random.choice(all_object_classes)
generated_no, generated_yes = False, False
question = None
temp_data = []
num_tries += 1
grid_file = 'layouts/%s-layout.npy' % scene_name
xray_graph = graph_obj.Graph(grid_file, use_gt=True, construct_graph=False)
scene_bounds = [xray_graph.xMin, xray_graph.yMin,
xray_graph.xMax - xray_graph.xMin + 1,
xray_graph.yMax - xray_graph.yMin + 1]
for i in range(20): # try 20 times
scene_seed = random.randint(0, 999999999)
episode.initialize_episode(scene_seed=scene_seed) # randomly initialize the scene
if not question:
question = ExistenceQuestion.get_true_contain_question(episode, all_object_classes, receptacles)
if DEBUG:
print(str(question))
if not question:
continue
object_class = question.object_class
parent_class = question.parent_object_class
answer = question.get_answer(episode)
object_target = constants.OBJECT_CLASS_TO_ID[object_class]
if answer and not generated_yes:
event = episode.event
xray_graph.memory[:, :, 1:] = 0
objs = {obj['objectId']: obj for obj in event.metadata['objects']
if obj['objectType'] == object_class and obj['parentReceptacle'].split('|')[0] == parent_class}
for obj in objs.values():
if obj['objectType'] != object_class:
continue
obj_point = game_util.get_object_point(obj, scene_bounds)
xray_graph.memory[obj_point[1], obj_point[0],
constants.OBJECT_CLASS_TO_ID[obj['objectType']] + 1] = 1
# Make sure findable
try:
graph_points = xray_graph.points.copy()
graph_points = graph_points[np.random.permutation(graph_points.shape[0]), :]
num_checked_points = 0
for start_point in graph_points:
headings = np.random.permutation(4)
for heading in headings:
start_point = (start_point[0], start_point[1], heading)
patch = xray_graph.get_graph_patch(start_point)[0]
if patch[:, :, object_target + 1].max() > 0:
action = {'action': 'TeleportFull',
'x': start_point[0] * constants.AGENT_STEP_SIZE,
'y': episode.agent_height,
'z': start_point[1] * constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'rotation': start_point[2] * 90,
'horizon': -30,
}
event = episode.env.step(action)
num_checked_points += 1
if num_checked_points > 1000:
answer = None
raise AssertionError
for jj in range(4):
open_success = True
opened_objects = set()
parents = [game_util.get_object(obj['parentReceptacle'], event.metadata)
for obj in objs.values()]
openable_parents = [parent for parent in parents
if parent['visible'] and parent['openable'] and not parent['isopen']]
while open_success:
for obj in objs.values():
if obj['objectId'] in event.instance_detections2D:
if game_util.check_object_size(event.instance_detections2D[obj['objectId']]):
raise AssertionError
if len(openable_parents) > 0:
action = {'action': 'OpenObject'}
game_util.set_open_close_object(action, event)
event = episode.env.step(action)
open_success = event.metadata['lastActionSuccess']
if open_success:
opened_objects.add(episode.env.last_action['objectId'])
else:
open_success = False
for opened in opened_objects:
event = episode.env.step({
'action': 'CloseObject',
'objectId': opened,
'forceVisible': True})
if not event.metadata['lastActionSuccess']:
answer = None
raise AssertionError
if jj < 3:
event = episode.env.step({'action': 'LookDown'})
answer = None
except AssertionError:
if answer is None and DEBUG:
print('failed to generate')
print(str(question), answer)
if answer == False and not generated_no:
generated_no = True
temp_data.append([scene_num, scene_seed, constants.OBJECT_CLASS_TO_ID[object_class], constants.OBJECT_CLASS_TO_ID[parent_class], answer])
elif answer == True and not generated_yes:
generated_yes = True
temp_data.append([scene_num, scene_seed, constants.OBJECT_CLASS_TO_ID[object_class], constants.OBJECT_CLASS_TO_ID[parent_class], answer])
if generated_no and generated_yes:
h5['questions/question'][data_ind, :] = np.array(temp_data[0])
h5['questions/question'][data_ind + 1, :] = np.array(temp_data[1])
h5.flush()
data_ind += 2
k += 2
break
print("# generated samples: {}".format(data_ind))
h5.close()
episode.env.stop_unity()
def create_dump():
time_str = py_util.get_time_str()
prefix = 'questions/'
if not os.path.exists(prefix + dataset_type + '/data_counting'):
os.makedirs(prefix + dataset_type + '/data_counting')
h5 = h5py.File(
prefix + dataset_type + '/data_counting/Counting_Questions_' +
time_str + '.h5', 'w')
h5.create_dataset('questions/question', (num_record, 4),
dtype=np.int32)
print('Generating %d counting questions' % num_record)
# Generate counting questions
data_ind = 0
episode = Episode()
scene_number = random.randint(0, len(scene_numbers) - 1)
while data_ind < num_record:
k = 0
scene_number += 1
scene_num = scene_numbers[scene_number % len(scene_numbers)]
scene_name = 'FloorPlan%d' % scene_num
episode.initialize_scene(scene_name)
num_tries = 0
while num_tries < num_samples_per_scene:
# randomly pick a pickable object in the scene
object_class = random.choice(all_object_classes)
question = CountQuestion(
object_class
) # randomly generate a general counting question
generated = [None] * (constants.MAX_COUNTING_ANSWER + 1)
generated_counts = set()
num_tries += 1
grid_file = 'layouts/%s-layout.npy' % scene_name
xray_graph = graph_obj.Graph(grid_file,
use_gt=True,
construct_graph=False)
scene_bounds = [
xray_graph.xMin, xray_graph.yMin,
xray_graph.xMax - xray_graph.xMin + 1,
xray_graph.yMax - xray_graph.yMin + 1
]
for i in range(100):
if DEBUG:
print('starting try ', i)
scene_seed = random.randint(0, 999999999)
episode.initialize_episode(
scene_seed=scene_seed,
max_num_repeats=constants.MAX_COUNTING_ANSWER + 1,
remove_prob=0.5)
answer = question.get_answer(episode)
object_target = constants.OBJECT_CLASS_TO_ID[object_class]
if answer > 0 and answer not in generated_counts:
if DEBUG:
print('target', str(question), object_target,
answer)
event = episode.event
# Make sure findable
try:
objs = {
obj['objectId']: obj
for obj in event.metadata['objects']
if obj['objectType'] == object_class
}
xray_graph.memory[:, :, 1:] = 0
for obj in objs.values():
obj_point = game_util.get_object_point(
obj, scene_bounds)
xray_graph.memory[obj_point[1], obj_point[0],
object_target + 1] = 1
start_graph = xray_graph.memory.copy()
graph_points = xray_graph.points.copy()
graph_points = graph_points[np.random.permutation(
graph_points.shape[0]), :]
num_checked_points = 0
point_ind = 0
# Initial check to make sure all objects are visible on the grid.
while point_ind < len(graph_points):
start_point = graph_points[point_ind]
headings = np.random.permutation(4)
for heading in headings:
start_point = (start_point[0],
start_point[1], heading)
patch = xray_graph.get_graph_patch(
start_point)[0][:, :,
object_target + 1]
if patch.max() > 0:
point_ind = 0
xray_graph.update_graph(
(np.zeros((constants.STEPS_AHEAD,
constants.STEPS_AHEAD,
1)), 0), start_point,
[object_target + 1])
point_ind += 1
if np.max(xray_graph.memory[:, :, object_target +
1]) > 0:
if DEBUG:
print('some points could not be reached')
answer = None
raise AssertionError
xray_graph.memory = start_graph
point_ind = 0
seen_objs = set()
while point_ind < len(graph_points):
start_point = graph_points[point_ind]
headings = np.random.permutation(4)
for heading in headings:
start_point = (start_point[0],
start_point[1], heading)
patch = xray_graph.get_graph_patch(
start_point)[0]
if patch[:, :,
object_target + 1].max() > 0:
action = {
'action': 'TeleportFull',
'x': start_point[0] *
constants.AGENT_STEP_SIZE,
'y': episode.agent_height,
'z': start_point[1] *
constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'rotation': start_point[2] * 90,
'horizon': -30,
}
event = episode.env.step(action)
num_checked_points += 1
if num_checked_points > 20:
if DEBUG:
print('timeout')
answer = None
raise AssertionError
changed = False
for jj in range(4):
open_success = True
opened_objects = set()
parents = [
game_util.get_object(
obj['parentReceptacle'],
event.metadata)
for obj in objs.values()
]
openable_parents = [
parent for parent in parents
if parent['visible']
and parent['openable']
and not parent['isopen']
]
while open_success:
obj_list = list(objs.values())
for obj in obj_list:
if obj['objectId'] in event.instance_detections2D:
if game_util.check_object_size(
event.
instance_detections2D[
obj['objectId']]
):
seen_objs.add(
obj['objectId']
)
if DEBUG:
print(
'seen',
seen_objs)
del objs[obj[
'objectId']]
changed = True
num_checked_points = 0
if len(seen_objs
) == answer:
raise AssertionError
if len(openable_parents) > 0:
action = {
'action': 'OpenObject'
}
game_util.set_open_close_object(
action, event)
event = episode.env.step(
action)
open_success = event.metadata[
'lastActionSuccess']
if open_success:
opened_objects.add(
episode.env.
last_action[
'objectId'])
else:
open_success = False
for opened in opened_objects:
event = episode.env.step({
'action':
'CloseObject',
'objectId':
opened,
'forceVisible':
True
})
if not event.metadata[
'lastActionSuccess']:
answer = None
raise AssertionError
if jj < 3:
event = episode.env.step(
{'action': 'LookDown'})
if changed:
point_ind = 0
num_checked_points = 0
xray_graph.memory[:, :,
object_target +
1] = 0
for obj in objs.values():
obj_point = game_util.get_object_point(
obj, scene_bounds)
xray_graph.memory[
obj_point[1], obj_point[0],
object_target + 1] = 1
point_ind += 1
if DEBUG:
print('ran out of points')
answer = None
except AssertionError:
if answer is not None:
if DEBUG:
print('success')
pass
print(str(question), object_target, answer)
if answer is not None and answer < len(
generated) and answer not in generated_counts:
generated[answer] = [
scene_num, scene_seed,
constants.OBJECT_CLASS_TO_ID[object_class], answer
]
generated_counts.add(answer)
print('\tcounts', sorted(list(generated_counts)))
if len(generated_counts) == len(generated):
for q in generated:
if data_ind >= h5['questions/question'].shape[0]:
num_tries = 2**32
break
h5['questions/question'][data_ind, :] = np.array(q)
data_ind += 1
k += 1
h5.flush()
break
print("# generated samples: {}".format(data_ind))
h5.close()
episode.env.stop_unity()