def test_random_initialize_randomize_open(env):
event = env.reset('FloorPlan1')
env.step(
dict(action='Initialize',
gridSize=0.25,
cameraY=0.75,
qualitySetting='High'))
event = env.random_initialize(random_seed=0, randomize_open=True)
event = env.step({'action': 'RotateRight'})
event = env.step({'action': 'RotateRight'})
event = assert_failed_action(env, {
'action': 'OpenObject',
'objectId': 'Microwave|-00.25|+01.71|-02.65'
})
visible_objects = {
obj['objectId']
for obj in event.metadata['objects'] if obj['visible']
}
assert (visible_objects == {
'Microwave|-00.25|+01.71|-02.65',
})
assert (game_util.get_object('Microwave|-00.25|+01.71|-02.65',
event.metadata)['isopen'])
event = assert_successful_action(
env, {
'action': 'CloseObject',
'objectId': 'Microwave|-00.25|+01.71|-02.65'
})
assert (not game_util.get_object('Microwave|-00.25|+01.71|-02.65',
event.metadata)['isopen'])
event = assert_successful_action(
env, {
'action': 'TeleportFull',
'x': -1,
'y': 1,
'z': 1,
'rotation': 270,
'horizon': 30
})
assert (game_util.get_object('Fridge|-01.89|+00.00|+01.07',
event.metadata)['isopen'])
event = assert_failed_action(env, {
'action': 'OpenObject',
'objectId': 'Fridge|-01.89|+00.00|+01.07'
})
visible_objects = {
obj['objectId']
for obj in event.metadata['objects'] if obj['visible']
}
assert (visible_objects == {
'Fridge|-01.89|+00.00|+01.07',
})
def get_parent_receps(self, object_id):
obj = game_util.get_object(object_id, self.env.last_event.metadata)
if obj is None or obj['parentReceptacles'] is None:
return None
parent_receps = [game_util.get_object(recep, self.env.last_event.metadata) for recep in
obj['parentReceptacles']]
parent_receps = [recep for recep in parent_receps if recep['objectType'] in constants.OPENABLE_CLASS_SET]
if len(parent_receps) > 0:
return parent_receps[0]
else:
return None
def process_frame(self):
self.pose = self.event.pose
self.s_t = self.event.frame
self.detection_image = self.s_t.copy()
self.s_t_depth = self.event.depth_frame
boxes = []
scores = []
class_names = []
for k, v in self.event.class_detections2D.items():
if k in constants.OBJECTS_SET:
boxes.extend(v)
scores.extend([1] * len(v))
class_names.extend([k] * len(v))
detected_objects = [
game_util.get_object(obj_id, self.event.metadata)
for obj_id in self.event.instance_detections2D.keys()
]
detected_objects = [obj for obj in detected_objects if obj is not None]
boxes = np.array([
self.event.instance_detections2D[obj['objectId']]
for obj in detected_objects
])
class_names = np.array([obj['objectType'] for obj in detected_objects])
scores = np.ones(len(boxes))
self.detection_image = drawing.visualize_detections(
self.event.frame, boxes, class_names, scores)
print(self.question_str)
def correct_slice_id(self, object_id):
main_obj = game_util.get_object(object_id, self.env.last_event.metadata)
if (main_obj is not None and
main_obj['objectType'] in constants.VAL_ACTION_OBJECTS['Sliceable'] and main_obj['isSliced']):
slice_obj_type = main_obj['objectType'] + "Sliced"
# Iterate through slices and pick up the one closest to the agent according to the 'distance' metadata.
min_d = None
best_slice_id = None
sidx = 1
while True:
slice_obj_id = main_obj['objectId'] + "|" + slice_obj_type + "_%d" % sidx
slice_obj = game_util.get_object(slice_obj_id, self.env.last_event.metadata)
if slice_obj is None:
break
if min_d is None or slice_obj['distance'] < min_d:
min_d = slice_obj['distance']
best_slice_id = slice_obj_id
sidx += 1
return best_slice_id
return object_id
def step(self, action_or_ind):
# refresh every step
self.update_receptacle_nearest_points()
action, should_fail = self.get_action(action_or_ind)
if 'objectId' in action:
assert isinstance(action['objectId'], str)
if 'receptacleObjectId' in action:
assert isinstance(action['receptacleObjectId'], str)
if action['action'] == 'PutObject' and self.env.last_event.metadata[
'lastActionSuccess']:
object_cls = constants.OBJECT_CLASS_TO_ID[action['objectId'].split(
'|')[0]]
receptacle_cls = constants.OBJECT_CLASS_TO_ID[
action['receptacleObjectId'].split('|')[0]]
if object_cls == self.object_target and receptacle_cls == self.parent_target:
pass
if constants.DEBUG:
print('step action', game_util.get_action_str(action))
GameStateBase.step(self, action_or_ind)
if action['action'] == 'PickupObject':
if 'receptacleObjectId' in action:
# Could be false in the case of slice
if action['objectId'] in self.in_receptacle_ids[
action['receptacleObjectId']]:
self.in_receptacle_ids[
action['receptacleObjectId']].remove(
action['objectId'])
elif action['action'] == 'PutObject':
key = action['receptacleObjectId']
assert isinstance(key, str)
if key not in self.in_receptacle_ids:
self.in_receptacle_ids[key] = set()
self.in_receptacle_ids[key].add(action['objectId'])
elif action['action'] == 'CleanObject':
if self.env.last_event.metadata['lastActionSuccess']:
self.cleaned_object_ids.add(action['objectId'])
elif action['action'] == 'HeatObject':
pass
elif action['action'] == "ToggleObject":
pass
elif action['action'] == 'CoolObject':
pass
elif action['action'] == 'SliceObject':
pass
visible_objects = self.event.instance_detections2D.keys(
) if self.event.instance_detections2D != None else []
for obj in visible_objects:
obj = game_util.get_object(obj, self.env.last_event.metadata)
if obj is None:
continue
cls = obj['objectType']
obj_id = obj['objectId']
if cls not in constants.OBJECTS_SET:
continue
# Instantiate list of the same shape as bounds3d but with min and max point set to obj position.
if type(obj['parentReceptacles']) is list:
if len(obj['parentReceptacles']) == 1:
parent = obj['parentReceptacles'][0]
if len(obj['parentReceptacles']) > 1:
print("Warning: selecting first parent of " +
str(obj_id) + " from list " +
str(obj['parentReceptacles']))
else:
parent = None
else:
parent = obj['parentReceptacles']
if parent is not None and len(parent) > 0:
# TODO (cleanup): remove hack
fix_basin = False
if parent.startswith('Sink') and not parent.endswith('Basin'):
parent = parent + "|SinkBasin"
fix_basin = True
elif parent.startswith(
'Bathtub') and not parent.endswith('Basin'):
parent = parent + "|BathtubBasin"
fix_basin = True
if fix_basin:
try:
parent = game_util.get_object(
parent, self.env.last_event.metadata)
except KeyError:
raise Exception('No object named %s in scene %s' %
(parent, self.scene_name))
else:
parent = game_util.get_object(parent,
self.env.last_event.metadata)
if not parent['openable'] or parent['isOpen']:
parent_receptacle = parent['objectId']
if parent_receptacle in self.in_receptacle_ids:
self.in_receptacle_ids[parent_receptacle].add(obj_id)
self.object_to_point[
obj_id] = self.receptacle_to_point[
parent_receptacle]
self.point_to_object[tuple(
self.receptacle_to_point[parent_receptacle].tolist(
))] = obj_id
self.need_plan_update = True
def create_dump():
time_str = py_util.get_time_str()
prefix = 'questions/'
if not os.path.exists(prefix + dataset_type + '/data_contains'):
os.makedirs(prefix + dataset_type + '/data_contains')
h5 = h5py.File(prefix + dataset_type + '/data_contains/Contains_Questions_' + time_str + '.h5', 'w')
h5.create_dataset('questions/question', (num_record, 5), dtype=np.int32)
print('Generating %d contains questions' % num_record)
# Generate contains questions
data_ind = 0
episode = Episode()
scene_number = -1
while data_ind < num_record:
k = 0
scene_number += 1
scene_num = scene_numbers[scene_number % len(scene_numbers)]
scene_name = 'FloorPlan%d' % scene_num
episode.initialize_scene(scene_name)
num_tries = 0
while k < num_samples_per_scene and num_tries < 10 * num_samples_per_scene:
# randomly pick a pickable object in the scene
object_class = random.choice(all_object_classes)
generated_no, generated_yes = False, False
question = None
temp_data = []
num_tries += 1
grid_file = 'layouts/%s-layout.npy' % scene_name
xray_graph = graph_obj.Graph(grid_file, use_gt=True, construct_graph=False)
scene_bounds = [xray_graph.xMin, xray_graph.yMin,
xray_graph.xMax - xray_graph.xMin + 1,
xray_graph.yMax - xray_graph.yMin + 1]
for i in range(20): # try 20 times
scene_seed = random.randint(0, 999999999)
episode.initialize_episode(scene_seed=scene_seed) # randomly initialize the scene
if not question:
question = ExistenceQuestion.get_true_contain_question(episode, all_object_classes, receptacles)
if DEBUG:
print(str(question))
if not question:
continue
object_class = question.object_class
parent_class = question.parent_object_class
answer = question.get_answer(episode)
object_target = constants.OBJECT_CLASS_TO_ID[object_class]
if answer and not generated_yes:
event = episode.event
xray_graph.memory[:, :, 1:] = 0
objs = {obj['objectId']: obj for obj in event.metadata['objects']
if obj['objectType'] == object_class and obj['parentReceptacle'].split('|')[0] == parent_class}
for obj in objs.values():
if obj['objectType'] != object_class:
continue
obj_point = game_util.get_object_point(obj, scene_bounds)
xray_graph.memory[obj_point[1], obj_point[0],
constants.OBJECT_CLASS_TO_ID[obj['objectType']] + 1] = 1
# Make sure findable
try:
graph_points = xray_graph.points.copy()
graph_points = graph_points[np.random.permutation(graph_points.shape[0]), :]
num_checked_points = 0
for start_point in graph_points:
headings = np.random.permutation(4)
for heading in headings:
start_point = (start_point[0], start_point[1], heading)
patch = xray_graph.get_graph_patch(start_point)[0]
if patch[:, :, object_target + 1].max() > 0:
action = {'action': 'TeleportFull',
'x': start_point[0] * constants.AGENT_STEP_SIZE,
'y': episode.agent_height,
'z': start_point[1] * constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'rotation': start_point[2] * 90,
'horizon': -30,
}
event = episode.env.step(action)
num_checked_points += 1
if num_checked_points > 1000:
answer = None
raise AssertionError
for jj in range(4):
open_success = True
opened_objects = set()
parents = [game_util.get_object(obj['parentReceptacle'], event.metadata)
for obj in objs.values()]
openable_parents = [parent for parent in parents
if parent['visible'] and parent['openable'] and not parent['isopen']]
while open_success:
for obj in objs.values():
if obj['objectId'] in event.instance_detections2D:
if game_util.check_object_size(event.instance_detections2D[obj['objectId']]):
raise AssertionError
if len(openable_parents) > 0:
action = {'action': 'OpenObject'}
game_util.set_open_close_object(action, event)
event = episode.env.step(action)
open_success = event.metadata['lastActionSuccess']
if open_success:
opened_objects.add(episode.env.last_action['objectId'])
else:
open_success = False
for opened in opened_objects:
event = episode.env.step({
'action': 'CloseObject',
'objectId': opened,
'forceVisible': True})
if not event.metadata['lastActionSuccess']:
answer = None
raise AssertionError
if jj < 3:
event = episode.env.step({'action': 'LookDown'})
answer = None
except AssertionError:
if answer is None and DEBUG:
print('failed to generate')
print(str(question), answer)
if answer == False and not generated_no:
generated_no = True
temp_data.append([scene_num, scene_seed, constants.OBJECT_CLASS_TO_ID[object_class], constants.OBJECT_CLASS_TO_ID[parent_class], answer])
elif answer == True and not generated_yes:
generated_yes = True
temp_data.append([scene_num, scene_seed, constants.OBJECT_CLASS_TO_ID[object_class], constants.OBJECT_CLASS_TO_ID[parent_class], answer])
if generated_no and generated_yes:
h5['questions/question'][data_ind, :] = np.array(temp_data[0])
h5['questions/question'][data_ind + 1, :] = np.array(temp_data[1])
h5.flush()
data_ind += 2
k += 2
break
print("# generated samples: {}".format(data_ind))
h5.close()
episode.env.stop_unity()
def step(self, action_or_ind, process_frame=True):
if self.event is None:
self.event = self.env.last_event
self.current_frame_count += 1
self.total_frame_count += 1
action, should_fail = self.get_action(action_or_ind)
if should_fail:
self.env.last_event.metadata['lastActionSuccess'] = False
else:
t_start = time.time()
start_pose = game_util.get_pose(self.event)
if 'action' not in action or action['action'] is None or action['action'] == 'None':
self.env.last_event.metadata['lastActionSuccess'] = True
else:
if constants.RECORD_VIDEO_IMAGES:
im_ind = len(glob.glob(constants.save_path + '/*.png'))
if 'Teleport' in action['action']:
position = self.env.last_event.metadata['agent']['position']
rotation = self.env.last_event.metadata['agent']['rotation']
start_horizon = self.env.last_event.metadata['agent']['cameraHorizon']
start_rotation = rotation['y']
if (np.abs(action['x'] - position['x']) > 0.001 or
np.abs(action['z'] - position['z']) > 0.001):
# Movement
for xx in np.arange(.1, 1, .1):
new_action = copy.deepcopy(action)
new_action['x'] = np.round(position['x'] * (1 - xx) + action['x'] * xx, 5)
new_action['z'] = np.round(position['z'] * (1 - xx) + action['z'] * xx, 5)
new_action['rotation'] = start_rotation
new_action['horizon'] = start_horizon
self.event = self.env.step(new_action)
cv2.imwrite(constants.save_path + '/%09d.png' % im_ind,
self.event.frame[:, :, ::-1])
depth_image = self.event.depth_frame
depth_image = depth_image * (255 / MAX_DEPTH)
depth_image = depth_image.astype(np.uint8)
cv2.imwrite(constants.save_depth_path + '/%09d.png' % im_ind, depth_image)
game_util.store_image_name('%09d.png' % im_ind) # eww... seriously need to clean this up
im_ind += 1
if np.abs(action['horizon'] - self.env.last_event.metadata['agent']['cameraHorizon']) > 0.001:
end_horizon = action['horizon']
for xx in np.arange(.1, 1, .1):
new_action = copy.deepcopy(action)
new_action['horizon'] = np.round(start_horizon * (1 - xx) + end_horizon * xx, 3)
new_action['rotation'] = start_rotation
self.event = self.env.step(new_action)
cv2.imwrite(constants.save_path + '/%09d.png' % im_ind,
self.event.frame[:, :, ::-1])
depth_image = self.event.depth_frame
depth_image = depth_image * (255 / MAX_DEPTH)
depth_image = depth_image.astype(np.uint8)
cv2.imwrite(constants.save_depth_path + '/%09d.png' % im_ind, depth_image)
game_util.store_image_name('%09d.png' % im_ind)
im_ind += 1
if np.abs(action['rotation'] - rotation['y']) > 0.001:
end_rotation = action['rotation']
for xx in np.arange(.1, 1, .1):
new_action = copy.deepcopy(action)
new_action['rotation'] = np.round(start_rotation * (1 - xx) + end_rotation * xx, 3)
self.event = self.env.step(new_action)
cv2.imwrite(constants.save_path + '/%09d.png' % im_ind,
self.event.frame[:, :, ::-1])
depth_image = self.event.depth_frame
depth_image = depth_image * (255 / MAX_DEPTH)
depth_image = depth_image.astype(np.uint8)
cv2.imwrite(constants.save_depth_path + '/%09d.png' % im_ind, depth_image)
game_util.store_image_name('%09d.png' % im_ind)
im_ind += 1
self.event = self.env.step(action)
elif 'MoveAhead' in action['action']:
self.store_ll_action(action)
self.save_image(1)
events = self.env.smooth_move_ahead(action)
for event in events:
im_ind = len(glob.glob(constants.save_path + '/*.png'))
cv2.imwrite(constants.save_path + '/%09d.png' % im_ind, event.frame[:, :, ::-1])
depth_image = event.depth_frame
depth_image = depth_image * (255 / MAX_DEPTH)
depth_image = depth_image.astype(np.uint8)
cv2.imwrite(constants.save_depth_path + '/%09d.png' % im_ind, depth_image)
game_util.store_image_name('%09d.png' % im_ind)
elif 'Rotate' in action['action']:
self.store_ll_action(action)
self.save_image(1)
events = self.env.smooth_rotate(action)
for event in events:
im_ind = len(glob.glob(constants.save_path + '/*.png'))
cv2.imwrite(constants.save_path + '/%09d.png' % im_ind, event.frame[:, :, ::-1])
depth_image = event.depth_frame
depth_image = depth_image * (255 / MAX_DEPTH)
depth_image = depth_image.astype(np.uint8)
cv2.imwrite(constants.save_depth_path + '/%09d.png' % im_ind, depth_image)
game_util.store_image_name('%09d.png' % im_ind)
elif 'Look' in action['action']:
self.store_ll_action(action)
self.save_image(1)
events = self.env.smooth_look(action)
for event in events:
im_ind = len(glob.glob(constants.save_path + '/*.png'))
cv2.imwrite(constants.save_path + '/%09d.png' % im_ind, event.frame[:, :, ::-1])
depth_image = event.depth_frame
depth_image = depth_image * (255 / MAX_DEPTH)
depth_image = depth_image.astype(np.uint8)
cv2.imwrite(constants.save_depth_path + '/%09d.png' % im_ind, depth_image)
game_util.store_image_name('%09d.png' % im_ind)
elif 'OpenObject' in action['action']:
open_action = dict(action=action['action'],
objectId=action['objectId'],
moveMagnitude=1.0)
self.store_ll_action(open_action)
self.save_act_image(open_action, dir=constants.BEFORE)
self.event = self.env.step(open_action)
self.save_act_image(open_action, dir=constants.AFTER)
self.check_action_success(self.event)
elif 'CloseObject' in action['action']:
close_action = dict(action=action['action'],
objectId=action['objectId'])
self.store_ll_action(close_action)
self.save_act_image(close_action, dir=constants.BEFORE)
self.event = self.env.step(close_action)
self.save_act_image(close_action, dir=constants.AFTER)
self.check_action_success(self.event)
elif 'PickupObject' in action['action']:
# [hack] correct object ids of slices
action['objectId'] = self.correct_slice_id(action['objectId'])
# open the receptacle if needed
parent_recep = self.get_parent_receps(action['objectId'])
if parent_recep is not None and parent_recep['objectType'] in constants.OPENABLE_CLASS_SET:
self.open_recep(parent_recep) # stores LL action
# close/open the object if needed
pickup_obj = game_util.get_object(action['objectId'], self.env.last_event.metadata)
if pickup_obj['objectType'] in constants.FORCED_OPEN_STATE_ON_PICKUP:
if pickup_obj['isOpen'] != constants.FORCED_OPEN_STATE_ON_PICKUP[pickup_obj['objectType']]:
if pickup_obj['isOpen']:
self.close_recep(pickup_obj) # stores LL action
else:
self.open_recep(pickup_obj) # stores LL action
# pick up the object
self.check_obj_visibility(action, min_pixels=10)
pickup_action = dict(action=action['action'],
objectId=action['objectId'])
self.store_ll_action(pickup_action)
self.save_act_image(pickup_action, dir=constants.BEFORE)
self.event = self.env.step(pickup_action)
self.save_act_image(pickup_action, dir=constants.AFTER)
self.check_action_success(self.event)
# close the receptacle if needed
if parent_recep is not None:
parent_recep = game_util.get_object(parent_recep['objectId'], self.env.last_event.metadata)
self.close_recep(parent_recep) # stores LL action
elif 'PutObject' in action['action']:
if len(self.env.last_event.metadata['inventoryObjects']) > 0:
inv_obj = self.env.last_event.metadata['inventoryObjects'][0]['objectId']
else:
raise RuntimeError("Taking 'PutObject' action with no held inventory object")
action['objectId'] = inv_obj
# open the receptacle if needed
parent_recep = game_util.get_object(action['receptacleObjectId'], self.env.last_event.metadata)
if parent_recep is not None and parent_recep['objectType'] in constants.OPENABLE_CLASS_SET:
self.open_recep(parent_recep) # stores LL action
# Open the parent receptacle of the movable receptacle target.
elif parent_recep['objectType'] in constants.MOVABLE_RECEPTACLES_SET:
movable_parent_recep_ids = parent_recep['parentReceptacles']
if movable_parent_recep_ids is not None and len(movable_parent_recep_ids) > 0:
print(parent_recep['objectId'], movable_parent_recep_ids) # DEBUG
movable_parent_recep = game_util.get_object(movable_parent_recep_ids[0],
self.env.last_event.metadata)
if movable_parent_recep['objectType'] in constants.OPENABLE_CLASS_SET:
self.open_recep(movable_parent_recep) # stores LL action
# put the object
put_action = dict(action=action['action'],
objectId=action['receptacleObjectId'],
forceAction=True,
placeStationary=True)
self.store_ll_action(put_action)
self.save_act_image(put_action, dir=constants.BEFORE)
self.event = self.env.step(put_action)
self.save_act_image(put_action, dir=constants.AFTER)
self.check_obj_visibility(action)
self.check_action_success(self.event)
# close the receptacle if needed
if parent_recep is not None:
parent_recep = game_util.get_object(parent_recep['objectId'], self.env.last_event.metadata)
self.close_recep(parent_recep) # stores LL action
elif 'CleanObject' in action['action']:
# put the object in the sink
sink_obj_id = self.get_some_visible_obj_of_name('SinkBasin')['objectId']
inv_obj = self.env.last_event.metadata['inventoryObjects'][0]
put_action = dict(action='PutObject',
objectId=sink_obj_id,
forceAction=True,
placeStationary=True)
self.store_ll_action(put_action)
self.save_act_image(put_action, dir=constants.BEFORE)
self.event = self.env.step(put_action)
self.save_act_image(put_action, dir=constants.AFTER)
self.check_obj_visibility(put_action)
self.check_action_success(self.event)
# turn on the tap
clean_action = copy.deepcopy(action)
clean_action['action'] = 'ToggleObjectOn'
clean_action['objectId'] = game_util.get_obj_of_type_closest_to_obj(
"Faucet", inv_obj['objectId'], self.env.last_event.metadata)['objectId']
self.store_ll_action(clean_action)
self.save_act_image(clean_action, dir=constants.BEFORE)
self.event = self.env.step({k: clean_action[k]
for k in ['action', 'objectId']})
self.save_act_image(clean_action, dir=constants.AFTER)
self.check_action_success(self.event)
# Need to delay one frame to let `isDirty` update on stream-affected.
self.env.noop()
# Call built-in 'CleanObject' THOR action on every object in the SinkBasin.
# This means we clean everything in the sink, rather than just the things that happen to touch
# the water stream, which is the default simulator behavior but looks weird for our purposes.
sink_basin_obj = game_util.get_obj_of_type_closest_to_obj(
"SinkBasin", clean_action['objectId'], self.env.last_event.metadata)
for in_sink_obj_id in sink_basin_obj['receptacleObjectIds']:
if (game_util.get_object(in_sink_obj_id, self.env.last_event.metadata)['dirtyable']
and game_util.get_object(in_sink_obj_id, self.env.last_event.metadata)['isDirty']):
self.event = self.env.step({'action': 'CleanObject',
'objectId': in_sink_obj_id})
# turn off the tap
close_action = copy.deepcopy(clean_action)
close_action['action'] = 'ToggleObjectOff'
self.store_ll_action(close_action)
self.save_act_image(close_action, dir=constants.BEFORE)
self.event = self.env.step({k: close_action[k]
for k in ['action', 'objectId']})
self.save_act_image(action, dir=constants.AFTER)
self.check_action_success(self.event)
# pick up the object from the sink
pickup_action = dict(action='PickupObject',
objectId=inv_obj['objectId'])
self.store_ll_action(pickup_action)
self.save_act_image(pickup_action, dir=constants.BEFORE)
self.event = self.env.step(pickup_action)
self.save_act_image(pickup_action, dir=constants.AFTER)
self.check_obj_visibility(pickup_action)
self.check_action_success(self.event)
elif 'HeatObject' in action['action']:
# open the microwave
microwave_obj_id = self.get_some_visible_obj_of_name('Microwave')['objectId']
microwave_obj = game_util.get_object(microwave_obj_id, self.env.last_event.metadata)
self.open_recep(microwave_obj)
# put the object in the microwave
inv_obj = self.env.last_event.metadata['inventoryObjects'][0]
put_action = dict(action='PutObject',
objectId=microwave_obj_id,
forceAction=True,
placeStationary=True)
self.store_ll_action(put_action)
self.save_act_image(put_action, dir=constants.BEFORE)
self.event = self.env.step(put_action)
self.save_act_image(put_action, dir=constants.AFTER)
self.check_obj_visibility(put_action)
self.check_action_success(self.event)
# close the microwave
microwave_obj = game_util.get_object(microwave_obj_id, self.env.last_event.metadata)
self.close_recep(microwave_obj)
# turn on the microwave
heat_action = copy.deepcopy(action)
heat_action['action'] = 'ToggleObjectOn'
heat_action['objectId'] = microwave_obj_id
self.store_ll_action(heat_action)
self.save_act_image(heat_action, dir=constants.BEFORE)
self.event = self.env.step({k: heat_action[k]
for k in ['action', 'objectId']})
self.save_act_image(heat_action, dir=constants.AFTER)
# turn off the microwave
stop_action = copy.deepcopy(heat_action)
stop_action['action'] = 'ToggleObjectOff'
self.store_ll_action(stop_action)
self.save_act_image(stop_action, dir=constants.BEFORE)
self.event = self.env.step({k: stop_action[k]
for k in ['action', 'objectId']})
self.save_act_image(stop_action, dir=constants.AFTER)
# open the microwave
microwave_obj = game_util.get_object(microwave_obj_id, self.env.last_event.metadata)
self.open_recep(microwave_obj)
# pick up the object from the microwave
pickup_action = dict(action='PickupObject',
objectId=inv_obj['objectId'])
self.store_ll_action(pickup_action)
self.save_act_image(pickup_action, dir=constants.BEFORE)
self.event = self.env.step(pickup_action)
self.save_act_image(pickup_action, dir=constants.AFTER)
self.check_obj_visibility(pickup_action)
self.check_action_success(self.event)
# close the microwave again
microwave_obj = game_util.get_object(microwave_obj_id, self.env.last_event.metadata)
self.close_recep(microwave_obj)
elif 'CoolObject' in action['action']:
# open the fridge
fridge_obj_id = self.get_some_visible_obj_of_name('Fridge')['objectId']
fridge_obj = game_util.get_object(fridge_obj_id, self.env.last_event.metadata)
self.open_recep(fridge_obj)
# put the object in the fridge
inv_obj = self.env.last_event.metadata['inventoryObjects'][0]
put_action = dict(action='PutObject',
objectId=fridge_obj_id,
forceAction=True,
placeStationary=True)
self.store_ll_action(put_action)
self.save_act_image(put_action, dir=constants.BEFORE)
self.event = self.env.step(put_action)
self.save_act_image(put_action, dir=constants.AFTER)
self.check_obj_visibility(put_action)
self.check_action_success(self.event)
# close and cool the object inside the frige
cool_action = dict(action='CloseObject',
objectId=action['receptacleObjectId'])
self.store_ll_action(cool_action)
self.save_act_image(action, dir=constants.BEFORE)
self.event = self.env.step(cool_action)
self.save_act_image(action, dir=constants.MIDDLE)
self.save_act_image(action, dir=constants.AFTER)
# open the fridge again
fridge_obj = game_util.get_object(fridge_obj_id, self.env.last_event.metadata)
self.open_recep(fridge_obj)
# pick up the object from the fridge
pickup_action = dict(action='PickupObject',
objectId=inv_obj['objectId'])
self.store_ll_action(pickup_action)
self.save_act_image(pickup_action, dir=constants.BEFORE)
self.event = self.env.step(pickup_action)
self.save_act_image(pickup_action, dir=constants.AFTER)
self.check_obj_visibility(pickup_action)
self.check_action_success(self.event)
# close the fridge again
fridge_obj = game_util.get_object(fridge_obj_id, self.env.last_event.metadata)
self.close_recep(fridge_obj)
elif 'ToggleObject' in action['action']:
on_action = dict(action=action['action'],
objectId=action['objectId'])
toggle_obj = game_util.get_object(action['objectId'], self.env.last_event.metadata)
on_action['action'] = 'ToggleObjectOff' if toggle_obj['isToggled'] else 'ToggleObjectOn'
self.store_ll_action(on_action)
self.save_act_image(on_action, dir=constants.BEFORE)
self.event = self.env.step(on_action)
self.save_act_image(on_action, dir=constants.AFTER)
self.check_action_success(self.event)
elif 'SliceObject' in action['action']:
# open the receptacle if needed
parent_recep = self.get_parent_receps(action['objectId'])
if parent_recep is not None and parent_recep['objectType'] in constants.OPENABLE_CLASS_SET:
self.open_recep(parent_recep)
# slice the object
slice_action = dict(action=action['action'],
objectId=action['objectId'])
self.store_ll_action(slice_action)
self.save_act_image(slice_action, dir=constants.BEFORE)
self.event = self.env.step(slice_action)
self.save_act_image(action, dir=constants.AFTER)
self.check_action_success(self.event)
# close the receptacle if needed
if parent_recep is not None:
parent_recep = game_util.get_object(parent_recep['objectId'], self.env.last_event.metadata)
self.close_recep(parent_recep) # stores LL action
else:
# check that the object to pick is visible in the camera frame
if action['action'] == 'PickupObject':
self.check_obj_visibility(action)
self.store_ll_action(action)
self.save_act_image(action, dir=constants.BEFORE)
self.event = self.env.step(action)
self.save_act_image(action, dir=constants.AFTER)
if action['action'] == 'PutObject':
self.check_obj_visibility(action)
else:
self.event = self.env.step(action)
new_pose = game_util.get_pose(self.event)
point_dists = np.sum(np.abs(self.gt_graph.points - np.array(new_pose)[:2]), axis=1)
if np.min(point_dists) > 0.0001:
print('Point teleport failure')
self.event = self.env.step({
'action': 'Teleport',
'x': start_pose[0] * constants.AGENT_STEP_SIZE,
'y': self.agent_height,
'z': start_pose[1] * constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'rotation': new_pose[2] * 90,
})
self.env.last_event.metadata['lastActionSuccess'] = False
self.timers[0, 0] += time.time() - t_start
self.timers[0, 1] += 1
if self.timers[0, 1] % 100 == 0:
print('env step time %.3f' % (self.timers[0, 0] / self.timers[0, 1]))
self.timers[0, :] = 0
if self.env.last_event.metadata['lastActionSuccess']:
if action['action'] == 'OpenObject':
self.currently_opened_object_ids.add(action['objectId'])
elif action['action'] == 'CloseObject':
self.currently_opened_object_ids.remove(action['objectId'])
elif action['action'] == 'PickupObject':
self.inventory_ids.add(action['objectId'])
elif action['action'] == 'PutObject':
self.inventory_ids.remove(action['objectId'])
if self.env.last_event.metadata['lastActionSuccess'] and process_frame:
self.process_frame()
def create_dump():
time_str = py_util.get_time_str()
prefix = 'questions/'
if not os.path.exists(prefix + dataset_type + '/data_counting'):
os.makedirs(prefix + dataset_type + '/data_counting')
h5 = h5py.File(
prefix + dataset_type + '/data_counting/Counting_Questions_' +
time_str + '.h5', 'w')
h5.create_dataset('questions/question', (num_record, 4),
dtype=np.int32)
print('Generating %d counting questions' % num_record)
# Generate counting questions
data_ind = 0
episode = Episode()
scene_number = random.randint(0, len(scene_numbers) - 1)
while data_ind < num_record:
k = 0
scene_number += 1
scene_num = scene_numbers[scene_number % len(scene_numbers)]
scene_name = 'FloorPlan%d' % scene_num
episode.initialize_scene(scene_name)
num_tries = 0
while num_tries < num_samples_per_scene:
# randomly pick a pickable object in the scene
object_class = random.choice(all_object_classes)
question = CountQuestion(
object_class
) # randomly generate a general counting question
generated = [None] * (constants.MAX_COUNTING_ANSWER + 1)
generated_counts = set()
num_tries += 1
grid_file = 'layouts/%s-layout.npy' % scene_name
xray_graph = graph_obj.Graph(grid_file,
use_gt=True,
construct_graph=False)
scene_bounds = [
xray_graph.xMin, xray_graph.yMin,
xray_graph.xMax - xray_graph.xMin + 1,
xray_graph.yMax - xray_graph.yMin + 1
]
for i in range(100):
if DEBUG:
print('starting try ', i)
scene_seed = random.randint(0, 999999999)
episode.initialize_episode(
scene_seed=scene_seed,
max_num_repeats=constants.MAX_COUNTING_ANSWER + 1,
remove_prob=0.5)
answer = question.get_answer(episode)
object_target = constants.OBJECT_CLASS_TO_ID[object_class]
if answer > 0 and answer not in generated_counts:
if DEBUG:
print('target', str(question), object_target,
answer)
event = episode.event
# Make sure findable
try:
objs = {
obj['objectId']: obj
for obj in event.metadata['objects']
if obj['objectType'] == object_class
}
xray_graph.memory[:, :, 1:] = 0
for obj in objs.values():
obj_point = game_util.get_object_point(
obj, scene_bounds)
xray_graph.memory[obj_point[1], obj_point[0],
object_target + 1] = 1
start_graph = xray_graph.memory.copy()
graph_points = xray_graph.points.copy()
graph_points = graph_points[np.random.permutation(
graph_points.shape[0]), :]
num_checked_points = 0
point_ind = 0
# Initial check to make sure all objects are visible on the grid.
while point_ind < len(graph_points):
start_point = graph_points[point_ind]
headings = np.random.permutation(4)
for heading in headings:
start_point = (start_point[0],
start_point[1], heading)
patch = xray_graph.get_graph_patch(
start_point)[0][:, :,
object_target + 1]
if patch.max() > 0:
point_ind = 0
xray_graph.update_graph(
(np.zeros((constants.STEPS_AHEAD,
constants.STEPS_AHEAD,
1)), 0), start_point,
[object_target + 1])
point_ind += 1
if np.max(xray_graph.memory[:, :, object_target +
1]) > 0:
if DEBUG:
print('some points could not be reached')
answer = None
raise AssertionError
xray_graph.memory = start_graph
point_ind = 0
seen_objs = set()
while point_ind < len(graph_points):
start_point = graph_points[point_ind]
headings = np.random.permutation(4)
for heading in headings:
start_point = (start_point[0],
start_point[1], heading)
patch = xray_graph.get_graph_patch(
start_point)[0]
if patch[:, :,
object_target + 1].max() > 0:
action = {
'action': 'TeleportFull',
'x': start_point[0] *
constants.AGENT_STEP_SIZE,
'y': episode.agent_height,
'z': start_point[1] *
constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'rotation': start_point[2] * 90,
'horizon': -30,
}
event = episode.env.step(action)
num_checked_points += 1
if num_checked_points > 20:
if DEBUG:
print('timeout')
answer = None
raise AssertionError
changed = False
for jj in range(4):
open_success = True
opened_objects = set()
parents = [
game_util.get_object(
obj['parentReceptacle'],
event.metadata)
for obj in objs.values()
]
openable_parents = [
parent for parent in parents
if parent['visible']
and parent['openable']
and not parent['isopen']
]
while open_success:
obj_list = list(objs.values())
for obj in obj_list:
if obj['objectId'] in event.instance_detections2D:
if game_util.check_object_size(
event.
instance_detections2D[
obj['objectId']]
):
seen_objs.add(
obj['objectId']
)
if DEBUG:
print(
'seen',
seen_objs)
del objs[obj[
'objectId']]
changed = True
num_checked_points = 0
if len(seen_objs
) == answer:
raise AssertionError
if len(openable_parents) > 0:
action = {
'action': 'OpenObject'
}
game_util.set_open_close_object(
action, event)
event = episode.env.step(
action)
open_success = event.metadata[
'lastActionSuccess']
if open_success:
opened_objects.add(
episode.env.
last_action[
'objectId'])
else:
open_success = False
for opened in opened_objects:
event = episode.env.step({
'action':
'CloseObject',
'objectId':
opened,
'forceVisible':
True
})
if not event.metadata[
'lastActionSuccess']:
answer = None
raise AssertionError
if jj < 3:
event = episode.env.step(
{'action': 'LookDown'})
if changed:
point_ind = 0
num_checked_points = 0
xray_graph.memory[:, :,
object_target +
1] = 0
for obj in objs.values():
obj_point = game_util.get_object_point(
obj, scene_bounds)
xray_graph.memory[
obj_point[1], obj_point[0],
object_target + 1] = 1
point_ind += 1
if DEBUG:
print('ran out of points')
answer = None
except AssertionError:
if answer is not None:
if DEBUG:
print('success')
pass
print(str(question), object_target, answer)
if answer is not None and answer < len(
generated) and answer not in generated_counts:
generated[answer] = [
scene_num, scene_seed,
constants.OBJECT_CLASS_TO_ID[object_class], answer
]
generated_counts.add(answer)
print('\tcounts', sorted(list(generated_counts)))
if len(generated_counts) == len(generated):
for q in generated:
if data_ind >= h5['questions/question'].shape[0]:
num_tries = 2**32
break
h5['questions/question'][data_ind, :] = np.array(q)
data_ind += 1
k += 1
h5.flush()
break
print("# generated samples: {}".format(data_ind))
h5.close()
episode.env.stop_unity()
