def create_dump():
time_str = py_util.get_time_str()
prefix = 'questions/'
if not os.path.exists(prefix + dataset_type + '/data_counting'):
os.makedirs(prefix + dataset_type + '/data_counting')
h5 = h5py.File(
prefix + dataset_type + '/data_counting/Counting_Questions_' +
time_str + '.h5', 'w')
h5.create_dataset('questions/question', (num_record, 4),
dtype=np.int32)
print('Generating %d counting questions' % num_record)
# Generate counting questions
data_ind = 0
episode = Episode()
scene_number = random.randint(0, len(scene_numbers) - 1)
while data_ind < num_record:
k = 0
scene_number += 1
scene_num = scene_numbers[scene_number % len(scene_numbers)]
scene_name = 'FloorPlan%d' % scene_num
episode.initialize_scene(scene_name)
num_tries = 0
while num_tries < num_samples_per_scene:
# randomly pick a pickable object in the scene
object_class = random.choice(all_object_classes)
question = CountQuestion(
object_class
) # randomly generate a general counting question
generated = [None] * (constants.MAX_COUNTING_ANSWER + 1)
generated_counts = set()
num_tries += 1
grid_file = 'layouts/%s-layout.npy' % scene_name
xray_graph = graph_obj.Graph(grid_file,
use_gt=True,
construct_graph=False)
scene_bounds = [
xray_graph.xMin, xray_graph.yMin,
xray_graph.xMax - xray_graph.xMin + 1,
xray_graph.yMax - xray_graph.yMin + 1
]
for i in range(100):
if DEBUG:
print('starting try ', i)
scene_seed = random.randint(0, 999999999)
episode.initialize_episode(
scene_seed=scene_seed,
max_num_repeats=constants.MAX_COUNTING_ANSWER + 1,
remove_prob=0.5)
answer = question.get_answer(episode)
object_target = constants.OBJECT_CLASS_TO_ID[object_class]
if answer > 0 and answer not in generated_counts:
if DEBUG:
print('target', str(question), object_target,
answer)
event = episode.event
# Make sure findable
try:
objs = {
obj['objectId']: obj
for obj in event.metadata['objects']
if obj['objectType'] == object_class
}
xray_graph.memory[:, :, 1:] = 0
for obj in objs.values():
obj_point = game_util.get_object_point(
obj, scene_bounds)
xray_graph.memory[obj_point[1], obj_point[0],
object_target + 1] = 1
start_graph = xray_graph.memory.copy()
graph_points = xray_graph.points.copy()
graph_points = graph_points[np.random.permutation(
graph_points.shape[0]), :]
num_checked_points = 0
point_ind = 0
# Initial check to make sure all objects are visible on the grid.
while point_ind < len(graph_points):
start_point = graph_points[point_ind]
headings = np.random.permutation(4)
for heading in headings:
start_point = (start_point[0],
start_point[1], heading)
patch = xray_graph.get_graph_patch(
start_point)[0][:, :,
object_target + 1]
if patch.max() > 0:
point_ind = 0
xray_graph.update_graph(
(np.zeros((constants.STEPS_AHEAD,
constants.STEPS_AHEAD,
1)), 0), start_point,
[object_target + 1])
point_ind += 1
if np.max(xray_graph.memory[:, :, object_target +
1]) > 0:
if DEBUG:
print('some points could not be reached')
answer = None
raise AssertionError
xray_graph.memory = start_graph
point_ind = 0
seen_objs = set()
while point_ind < len(graph_points):
start_point = graph_points[point_ind]
headings = np.random.permutation(4)
for heading in headings:
start_point = (start_point[0],
start_point[1], heading)
patch = xray_graph.get_graph_patch(
start_point)[0]
if patch[:, :,
object_target + 1].max() > 0:
action = {
'action': 'TeleportFull',
'x': start_point[0] *
constants.AGENT_STEP_SIZE,
'y': episode.agent_height,
'z': start_point[1] *
constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'rotation': start_point[2] * 90,
'horizon': -30,
}
event = episode.env.step(action)
num_checked_points += 1
if num_checked_points > 20:
if DEBUG:
print('timeout')
answer = None
raise AssertionError
changed = False
for jj in range(4):
open_success = True
opened_objects = set()
parents = [
game_util.get_object(
obj['parentReceptacle'],
event.metadata)
for obj in objs.values()
]
openable_parents = [
parent for parent in parents
if parent['visible']
and parent['openable']
and not parent['isopen']
]
while open_success:
obj_list = list(objs.values())
for obj in obj_list:
if obj['objectId'] in event.instance_detections2D:
if game_util.check_object_size(
event.
instance_detections2D[
obj['objectId']]
):
seen_objs.add(
obj['objectId']
)
if DEBUG:
print(
'seen',
seen_objs)
del objs[obj[
'objectId']]
changed = True
num_checked_points = 0
if len(seen_objs
) == answer:
raise AssertionError
if len(openable_parents) > 0:
action = {
'action': 'OpenObject'
}
game_util.set_open_close_object(
action, event)
event = episode.env.step(
action)
open_success = event.metadata[
'lastActionSuccess']
if open_success:
opened_objects.add(
episode.env.
last_action[
'objectId'])
else:
open_success = False
for opened in opened_objects:
event = episode.env.step({
'action':
'CloseObject',
'objectId':
opened,
'forceVisible':
True
})
if not event.metadata[
'lastActionSuccess']:
answer = None
raise AssertionError
if jj < 3:
event = episode.env.step(
{'action': 'LookDown'})
if changed:
point_ind = 0
num_checked_points = 0
xray_graph.memory[:, :,
object_target +
1] = 0
for obj in objs.values():
obj_point = game_util.get_object_point(
obj, scene_bounds)
xray_graph.memory[
obj_point[1], obj_point[0],
object_target + 1] = 1
point_ind += 1
if DEBUG:
print('ran out of points')
answer = None
except AssertionError:
if answer is not None:
if DEBUG:
print('success')
pass
print(str(question), object_target, answer)
if answer is not None and answer < len(
generated) and answer not in generated_counts:
generated[answer] = [
scene_num, scene_seed,
constants.OBJECT_CLASS_TO_ID[object_class], answer
]
generated_counts.add(answer)
print('\tcounts', sorted(list(generated_counts)))
if len(generated_counts) == len(generated):
for q in generated:
if data_ind >= h5['questions/question'].shape[0]:
num_tries = 2**32
break
h5['questions/question'][data_ind, :] = np.array(q)
data_ind += 1
k += 1
h5.flush()
break
print("# generated samples: {}".format(data_ind))
h5.close()
episode.env.stop_unity()
def create_dump():
time_str = py_util.get_time_str()
prefix = 'questions/'
if not os.path.exists(prefix + dataset_type + '/data_contains'):
os.makedirs(prefix + dataset_type + '/data_contains')
h5 = h5py.File(prefix + dataset_type + '/data_contains/Contains_Questions_' + time_str + '.h5', 'w')
h5.create_dataset('questions/question', (num_record, 5), dtype=np.int32)
print('Generating %d contains questions' % num_record)
# Generate contains questions
data_ind = 0
episode = Episode()
scene_number = -1
while data_ind < num_record:
k = 0
scene_number += 1
scene_num = scene_numbers[scene_number % len(scene_numbers)]
scene_name = 'FloorPlan%d' % scene_num
episode.initialize_scene(scene_name)
num_tries = 0
while k < num_samples_per_scene and num_tries < 10 * num_samples_per_scene:
# randomly pick a pickable object in the scene
object_class = random.choice(all_object_classes)
generated_no, generated_yes = False, False
question = None
temp_data = []
num_tries += 1
grid_file = 'layouts/%s-layout.npy' % scene_name
xray_graph = graph_obj.Graph(grid_file, use_gt=True, construct_graph=False)
scene_bounds = [xray_graph.xMin, xray_graph.yMin,
xray_graph.xMax - xray_graph.xMin + 1,
xray_graph.yMax - xray_graph.yMin + 1]
for i in range(20): # try 20 times
scene_seed = random.randint(0, 999999999)
episode.initialize_episode(scene_seed=scene_seed) # randomly initialize the scene
if not question:
question = ExistenceQuestion.get_true_contain_question(episode, all_object_classes, receptacles)
if DEBUG:
print(str(question))
if not question:
continue
object_class = question.object_class
parent_class = question.parent_object_class
answer = question.get_answer(episode)
object_target = constants.OBJECT_CLASS_TO_ID[object_class]
if answer and not generated_yes:
event = episode.event
xray_graph.memory[:, :, 1:] = 0
objs = {obj['objectId']: obj for obj in event.metadata['objects']
if obj['objectType'] == object_class and obj['parentReceptacle'].split('|')[0] == parent_class}
for obj in objs.values():
if obj['objectType'] != object_class:
continue
obj_point = game_util.get_object_point(obj, scene_bounds)
xray_graph.memory[obj_point[1], obj_point[0],
constants.OBJECT_CLASS_TO_ID[obj['objectType']] + 1] = 1
# Make sure findable
try:
graph_points = xray_graph.points.copy()
graph_points = graph_points[np.random.permutation(graph_points.shape[0]), :]
num_checked_points = 0
for start_point in graph_points:
headings = np.random.permutation(4)
for heading in headings:
start_point = (start_point[0], start_point[1], heading)
patch = xray_graph.get_graph_patch(start_point)[0]
if patch[:, :, object_target + 1].max() > 0:
action = {'action': 'TeleportFull',
'x': start_point[0] * constants.AGENT_STEP_SIZE,
'y': episode.agent_height,
'z': start_point[1] * constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'rotation': start_point[2] * 90,
'horizon': -30,
}
event = episode.env.step(action)
num_checked_points += 1
if num_checked_points > 1000:
answer = None
raise AssertionError
for jj in range(4):
open_success = True
opened_objects = set()
parents = [game_util.get_object(obj['parentReceptacle'], event.metadata)
for obj in objs.values()]
openable_parents = [parent for parent in parents
if parent['visible'] and parent['openable'] and not parent['isopen']]
while open_success:
for obj in objs.values():
if obj['objectId'] in event.instance_detections2D:
if game_util.check_object_size(event.instance_detections2D[obj['objectId']]):
raise AssertionError
if len(openable_parents) > 0:
action = {'action': 'OpenObject'}
game_util.set_open_close_object(action, event)
event = episode.env.step(action)
open_success = event.metadata['lastActionSuccess']
if open_success:
opened_objects.add(episode.env.last_action['objectId'])
else:
open_success = False
for opened in opened_objects:
event = episode.env.step({
'action': 'CloseObject',
'objectId': opened,
'forceVisible': True})
if not event.metadata['lastActionSuccess']:
answer = None
raise AssertionError
if jj < 3:
event = episode.env.step({'action': 'LookDown'})
answer = None
except AssertionError:
if answer is None and DEBUG:
print('failed to generate')
print(str(question), answer)
if answer == False and not generated_no:
generated_no = True
temp_data.append([scene_num, scene_seed, constants.OBJECT_CLASS_TO_ID[object_class], constants.OBJECT_CLASS_TO_ID[parent_class], answer])
elif answer == True and not generated_yes:
generated_yes = True
temp_data.append([scene_num, scene_seed, constants.OBJECT_CLASS_TO_ID[object_class], constants.OBJECT_CLASS_TO_ID[parent_class], answer])
if generated_no and generated_yes:
h5['questions/question'][data_ind, :] = np.array(temp_data[0])
h5['questions/question'][data_ind + 1, :] = np.array(temp_data[1])
h5.flush()
data_ind += 2
k += 2
break
print("# generated samples: {}".format(data_ind))
h5.close()
episode.env.stop_unity()
def create_dump():
episode = Episode()
count = 0
#for i in range(len(scene_numbers)):
while count > -1:
scene_name = 'FloorPlan%d' % scene_numbers[count]
print('scene is %s' % scene_name)
grid_file = 'layouts/%s-layout.npy' % scene_name
graph = graph_obj.Graph(grid_file) #, use_gt=use_gt)
print("graph: ", graph)
scene_bounds = [
graph.xMin, graph.yMin, graph.xMax - graph.xMin + 1,
graph.yMax - graph.yMin + 1
]
print("bounds: ", scene_bounds)
episode.initialize_scene(scene_name)
#scene_seed = random.randint(0, 999999999)
#episode.initialize_episode(scene_seed=scene_seed) # randomly initialize the scene
prefix = 'object_semantic_map/maps/'
for index in range(1, 11):
#memory = np.zeros((graph.yMax - graph.yMin + 1, graph.xMax - graph.xMin + 1, 1 + constants.NUM_CLASSES))
npy_path = os.path.join(prefix, str(index) + '.npy')
memory = np.load(npy_path)
print(memory.shape)
"""
for _ in range(SIM_TIMES):
scene_seed = random.randint(0, 999999999)
episode.initialize_episode(scene_seed=scene_seed)
for obj in episode.get_objects():
if obj['objectType'] not in constants.OBJECTS:
continue
y, x = game_util.get_object_point(obj, scene_bounds)
#print("object %s at " % obj['objectType'], game_util.get_object_point(obj, scene_bounds))
obj_id = constants.OBJECT_CLASS_TO_ID[obj['objectType']]
memory[y][x][obj_id] += 1
memory = np.divide(memory, SIM_TIMES)
"""
print(memory.shape[:2])
#print (constants.OBJECTS[test_id])
#plt.figure(figsize=list(memory.shape[:2]))
#plt.pcolor(memory[:,:,test_id],cmap=plt.get_cmap('Reds'), vmin=0.0, vmax=1.0)#,edgecolors='k', linewidths=1)
#plt.colorbar()
#print (memory[:,:,test_id])
#plt.show()
fig, axs = plt.subplots(4, 7)
#a1
for (i, j) in list(itertools.product("0123", "23456")):
#print (i, j)
obj_id = (int(i)) * 5 + int(j) - 1
#if obj_id > 20: break
ax = axs[int(i), int(j)]
pcm = ax.pcolor(memory[:, :, obj_id],
cmap=plt.get_cmap('Reds'),
vmin=-0,
vmax=1)
ax.set_title(constants.OBJECTS[obj_id])
ax.axis('off')
gs = axs[0, 0].get_gridspec()
for ax in axs[:, :2].flatten():
ax.remove()
ax_frame = fig.add_subplot(gs[:, :2])
ax_frame.axis('off')
new_frame = get_agent_map_data(episode.env)
new_frame = new_frame[:, ::-1]
new_frame = np.rot90(new_frame, -1)
plt.imshow(new_frame)
#fig.tight_layout()
#fig.colorbar(pcm, ax=axs[:])
fig_path = os.path.join(prefix, str(index) + '.png')
plt.savefig(fig_path)
"""
objs = [obj['objectType'] for obj in episode.get_objects()]
objjs = [obj['objectId'] for obj in episode.get_objects()]
objjspos = [obj['position'] for obj in episode.get_objects()]
for ii in range(len(episode.get_objects())):
print("object%d at " % ii, game_util.get_object_point(episode.get_objects()[ii], scene_bounds))
print("objs: ", objs)
print("objjs: ", objjs)
print("objjs: ", len(objjs))
print("objjspos: ", objjspos)
print("objjspos: ", len(objjspos))
"""
#event = episode.get_env_top_view()
#print (event.metadata)
#image = drawing.subplot()
break