def test_random_initialize_speed(env):
print('Reset/Initialize speed')
t_start = time.time()
time_count = 0
for scene in range(1,31):
time_count += 1
game_util.reset(env, scene)
event = env.random_initialize(random_seed=0)
t_end = time.time()
total_time = t_end - t_start
print('Total time %.3f' % (total_time))
print('Mean time %.3f' % (total_time / time_count))
print('FPS = %.3f' % (time_count / total_time))
print('')
def initialize_episode(self,
scene_seed=None,
agent_seed=None,
max_num_repeats=10,
remove_prob=0.25):
"""Initializes environment with given scene and random seed."""
# Reset the scene with some random seed.
if scene_seed is None:
scene_seed = random.randint(0, 999999999)
if agent_seed is None:
agent_seed = random.randint(0, 999999999)
self.event = game_util.reset(self.env,
self.scene_name,
render_depth_image=False,
render_class_image=False,
render_object_image=True)
self.event = self.env.random_initialize(
random_seed=scene_seed,
max_num_repeats=max_num_repeats,
remove_prob=remove_prob)
self.agent_height = self.event.metadata['agent']['position']['y']
self.is_initialized = True
return scene_seed, agent_seed
def test_movement_speed(env):
print('Movement speed')
time_count = 0
total_time = 0
for scene in range(1,31):
game_util.reset(env, scene)
event = env.random_initialize(random_seed=0)
t_start = time.time()
for jj in range(10):
event = env.step({'action': 'MoveAhead'})
event = env.step({'action': 'RotateRight'})
time_count += 2
total_time += time.time() - t_start
print('Total time %.3f' % (total_time))
print('Mean time %.3f' % (total_time / time_count))
print('FPS = %.3f' % (time_count / total_time))
print('')
def get_env_info(self):
"""Get env specific information."""
event = game_util.reset(self.env,
self.scene_name,
render_depth_image=False,
render_class_image=False,
render_object_image=True)
self.object_id_to_object_class = {
obj['objectId']: obj['objectType']
for obj in event.metadata['objects']
}
self.pickable_object_classes = sorted(
list(
set([
obj['objectType'] for obj in event.metadata['objects']
if obj['pickupable'] and obj['objectType'] != 'MiscObject'
])))
print('# Pickable object_classes:', len(self.pickable_object_classes))
# Find all receptacles.
self.receptacles = sorted([
obj['objectId'] for obj in event.metadata['objects']
if obj['receptacle']
])
print('# Receptacles:', len(self.receptacles))
# Find all receptacle classes.
self.receptacle_classes = list(
set([item.split('|')[0] for item in self.receptacles]))
print('# Receptacle classes:', len(self.receptacle_classes))
# Find all openable receptacles.
self.openable_receptacles = sorted([
obj['objectId'] for obj in event.metadata['objects']
if obj['receptacle'] and obj['openable']
])
print('# Openable Receptacles:', len(self.openable_receptacles))
# Find all openable receptacle classes.
self.openable_receptacle_classes = list(
set([item.split('|')[0] for item in self.openable_receptacles]))
print('# Openable object_classes:',
len(self.openable_receptacle_classes))
self.agent_height = event.metadata['agent']['position']['y']
def reset(self, seed=None, test_ind=None):
self.board = None
self.seen_object = False
self.terminal = False
self.opened_receptacles = set()
self.closed_receptacles = set()
self.seen_obj1 = set()
self.seen_obj2 = set()
self.visited_locations = set()
self.can_end = False
if seed is not None:
self.local_random.seed(seed)
# Do equal number of each question type in train.
question_type_ind = self.local_random.sample(
constants.USED_QUESTION_TYPES, 1)[0]
# get random row
if test_ind is not None:
question_row, question_type_ind = test_ind
question_type = self.question_types[question_type_ind]
question_data = self.test_datasets[question_type_ind][
question_row, :]
test_ind = (question_row, question_type_ind)
else:
question_type = self.question_types[question_type_ind]
question_row = self.local_random.randint(
0,
len(self.datasets[question_type_ind]) - 1)
question_data = self.datasets[question_type_ind][question_row, :]
container_ind = None
if question_type_ind == 0 or question_type_ind == 1:
scene_num, scene_seed, object_ind, answer = question_data
self.question_target = object_ind
if question_type_ind == 0:
answer = bool(answer)
elif question_type_ind == 2:
scene_num, scene_seed, object_ind, container_ind, answer = question_data
answer = bool(answer)
self.question_target = (object_ind, container_ind)
else:
raise Exception('No question type found for type %d' %
question_type_ind)
self.scene_seed = scene_seed
self.scene_num = scene_num
self.object_target = object_ind
self.parent_target = container_ind
self.container_target = np.zeros(constants.NUM_CLASSES)
self.direction_target = np.zeros(4)
if container_ind is not None:
self.container_target[container_ind] = 1
self.question_type_ind = question_type_ind
self.scene_name = 'FloorPlan%d' % scene_num
grid_file = 'layouts/%s-layout.npy' % self.scene_name
self.graph = graph_obj.Graph(grid_file, use_gt=False)
self.xray_graph = graph_obj.Graph(grid_file, use_gt=True)
self.bounds = [
self.graph.xMin, self.graph.yMin,
self.graph.xMax - self.graph.xMin + 1,
self.graph.yMax - self.graph.yMin + 1
]
max_num_repeats = 1
remove_prob = 0.5
if question_type == 'existence':
max_num_repeats = 10
remove_prob = 0.25
elif question_type == 'counting':
max_num_repeats = constants.MAX_COUNTING_ANSWER + 1
remove_prob = 0.5
elif question_type == 'contains':
max_num_repeats = 10
remove_prob = 0.25
self.event = game_util.reset(self.env, self.scene_name)
self.agent_height = self.event.metadata['agent']['position']['y']
self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET
self.event = self.env.random_initialize(
self.scene_seed,
max_num_repeats=max_num_repeats,
remove_prob=remove_prob)
print('Type:', question_type, 'Row: ', question_row, 'Scene',
self.scene_name, 'seed', scene_seed)
print(
'Question:',
game_util.get_question_str(question_type_ind, object_ind,
container_ind))
if self.question_type_ind == 2:
print('Answer:', constants.OBJECTS[object_ind], 'in',
constants.OBJECTS[container_ind], 'is', answer)
else:
print('Answer:', constants.OBJECTS[object_ind], 'is', answer)
self.answer = answer
# Verify answer
if self.question_type_ind == 0:
objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind],
self.event.metadata)
computed_answer = len(objs) > 0
requires_interaction = True
for obj in objs:
parent = obj['parentReceptacle'].split('|')[0]
if parent not in {'Fridge', 'Cabinet', 'Microwave'}:
requires_interaction = False
break
elif self.question_type_ind == 1:
objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind],
self.event.metadata)
computed_answer = len(objs)
requires_interaction = True
elif self.question_type_ind == 2:
objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind],
self.event.metadata)
if len(objs) == 0:
computed_answer = False
requires_interaction = constants.OBJECTS[
self.question_target[1]] in {
'Fridge', 'Cabinet', 'Microwave'
}
else:
obj = objs[0]
computed_answer = False
for obj in objs:
requires_interaction = True
parent = obj['parentReceptacle'].split('|')[0]
if parent in constants.OBJECT_CLASS_TO_ID:
parent_ind = constants.OBJECT_CLASS_TO_ID[parent]
computed_answer = parent_ind == self.question_target[1]
if computed_answer:
if parent not in {
'Fridge', 'Cabinet', 'Microwave'
}:
requires_interaction = False
break
else:
computed_answer = False
self.requires_interaction = requires_interaction
try:
assert self.answer == computed_answer, 'Answer does not match scene metadata'
except AssertionError:
print('Type:', question_type, 'Row: ', question_row, 'Scene',
self.scene_name, 'seed', scene_seed)
print('Answer', computed_answer, 'does not match expected value',
self.answer, ', did randomization process change?')
pdb.set_trace()
self.answer = computed_answer
if constants.NUM_CLASSES > 1:
self.hidden_items = set()
objects = self.event.metadata['objects']
for obj in objects:
if obj['receptacle'] and obj['openable'] and not obj['isopen']:
for inside_obj in obj['receptacleObjectIds']:
self.hidden_items.add(inside_obj)
objects = self.event.metadata['objects']
for obj in objects:
if obj['objectType'] not in constants.OBJECT_CLASS_TO_ID:
continue
obj_bounds = game_util.get_object_bounds(obj, self.bounds)
self.xray_graph.memory[
obj_bounds[1]:obj_bounds[3], obj_bounds[0]:obj_bounds[2],
constants.OBJECT_CLASS_TO_ID[obj['objectType']] + 1] = 1
start_point = self.local_random.randint(0,
self.graph.points.shape[0] - 1)
start_point = self.graph.points[start_point, :].copy()
self.start_point = (start_point[0], start_point[1],
self.local_random.randint(0, 3))
action = {
'action': 'TeleportFull',
'x': self.start_point[0] * constants.AGENT_STEP_SIZE,
'y': self.agent_height,
'z': self.start_point[1] * constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'rotation': self.start_point[2] * 90,
'horizon': 30,
}
self.event = self.env.step(action)
self.process_frame()
self.reward = 0
self.end_point = []
def reset(self,
scene_name=None,
use_gt=True,
seed=None,
config_filename=""):
if scene_name is None:
# Do half reset
action_ind = self.local_random.randint(
0, constants.STEPS_AHEAD**2 - 1)
action_x = action_ind % constants.STEPS_AHEAD - int(
constants.STEPS_AHEAD / 2)
action_z = int(action_ind / constants.STEPS_AHEAD) + 1
x_shift = 0
z_shift = 0
if self.pose[2] == 0:
x_shift = action_x
z_shift = action_z
elif self.pose[2] == 1:
x_shift = action_z
z_shift = -action_x
elif self.pose[2] == 2:
x_shift = -action_x
z_shift = -action_z
elif self.pose[2] == 3:
x_shift = -action_z
z_shift = action_x
action_x = self.pose[0] + x_shift
action_z = self.pose[1] + z_shift
self.end_point = (action_x, action_z, self.pose[2])
#print ("in the game state reset end point is : ", self.end_point)
else:
# Do full reset
self.scene_name = scene_name
#print ("Full reset - in the first time of load")
grid_file = 'layouts/%s-layout_%s.npy' % (
scene_name, str(constants.AGENT_STEP_SIZE))
self.graph = graph_obj.Graph(grid_file,
self.action_util,
use_gt=use_gt)
if seed is not None:
self.local_random.seed(seed)
lastActionSuccess = False
self.discovered_explored = {}
self.discovered_objects = []
self.bounds = [
self.graph.xMin, self.graph.yMin,
self.graph.xMax - self.graph.xMin + 1,
self.graph.yMax - self.graph.yMin + 1
]
'''
while not lastActionSuccess:
print ("In the while loop")
self.event = game_util.reset(self.env, self.scene_name)
self.event = self.event.events[0]
self.agent_height = self.event.metadata['agent']['position']['y']
self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET
#self.event = self.env.random_initialize(seed)
#self.start_point = []
start_point_2 = []
#start_point_2.append(int(self.event.metadata['agent']['position']['x']/constants.AGENT_STEP_SIZE) -self.graph.xMin +1)
#start_point_2.append(int(self.event.metadata['agent']['position']['z']/constants.AGENT_STEP_SIZE) - self.graph.yMin +1 )
start_point_2.append(int(self.event.metadata['agent']['position']['x']/constants.AGENT_STEP_SIZE) )
start_point_2.append(int(self.event.metadata['agent']['position']['z']/constants.AGENT_STEP_SIZE) )
start_point_2.append(int(self.event.metadata['agent']['rotation']['y']/90))
print(start_point_2)
print ("starting horizon", self.event.metadata['agent']['cameraHorizon'])
self.start_point = start_point_2[:]
#start_point = self.local_random.randint(0, self.graph.points.shape[0] - 1)
#start_point = self.graph.points[start_point, :].copy()
#self.start_point = (start_point[0], start_point[1], self.local_random.randint(0, 3))
self.end_point = self.start_point
#print ("B4 the while loop for assigning end point", self.start_point)
while self.end_point[0] == self.start_point[0] and self.end_point[1] == self.start_point[1]:
#end_point = self.local_random.randint(0, self.graph.points.shape[0] - 1)
#end_point = self.graph.points[end_point, :].copy()
#self.end_point = [end_point[0], end_point[1], self.local_random.randint(0, 3)]
#self.end_point[0] += self.local_random.randint(-constants.TERMINAL_CHECK_PADDING, constants.TERMINAL_CHECK_PADDING)
#self.end_point[1] += self.local_random.randint(-constants.TERMINAL_CHECK_PADDING, constants.TERMINAL_CHECK_PADDING)
#self.end_point = tuple(self.end_point)
self.end_point = (self.end_point[0]-1 , self.end_point[1]-1 , self.end_point[2])
print ("In the while loop for assigning end point",self.end_point)
action = {'action': 'TeleportFull',
'x': self.start_point[0] * constants.AGENT_STEP_SIZE,
'y': self.agent_height,
'z': self.start_point[1] * constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'rotation': self.start_point[2] * 90,
'horizon': 0}
#'horizon': 60}
'''
while True:
self.event = game_util.reset(self.env, self.scene_name,
config_filename)
#self.event = self.event.events[0]
print("type of event 2 : ", type(self.event))
lastActionSuccess = self.event.return_status
break
start_point_2 = []
'''
self.agent_height = self.event.metadata['agent']['position']['y']
self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET
start_point_2.append(int(self.event.metadata['agent']['position']['x']/constants.AGENT_STEP_SIZE) )
start_point_2.append(int(self.event.metadata['agent']['position']['z']/constants.AGENT_STEP_SIZE) )
start_point_2.append(int(self.event.metadata['agent']['rotation']['y']/90))
'''
print("rotation ", self.event.rotation)
rotation_angle = -(self.event.rotation % 90)
action = "RotateLook, rotation=%d" % int(rotation_angle)
self.goal = self.event.goal
#action = {'action':"Pass"}
#print ("z before turning ", self.event.position['z']/constants.AGENT_STEP_SIZE)
#print ("z before turning ", self.event.position['z'])
self.event = self.env.step(action)
#print ("z after turning ", self.event.position['z']/constants.AGENT_STEP_SIZE)
#print ("z after turning ", self.event.position['z'])
self.agent_height = self.event.position['y']
self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET
start_point_2.append(
math.floor(self.event.position['x'] /
constants.AGENT_STEP_SIZE))
start_point_2.append(
math.floor(self.event.position['z'] /
constants.AGENT_STEP_SIZE))
start_point_2.append(int(self.event.rotation / 90))
#print(start_point_2)
self.start_point = start_point_2[:]
self.end_point = self.start_point[:]
self.end_point = (self.end_point[0], self.end_point[1],
(self.end_point[2] + 1) % 4)
#print ("end point = ", self.end_point)
#print ("start point = ", self.start_point)
#print ("last action success", self.event['lastActionSuccess'])
#print (self.event)
#print ("last action success", self.event.lastActionSuccess)
#print ("number of objects : ",len(self.event.metadata['objects']))
#action = {"action": "RotateRight", 'rotatio#n':90}
#print ("type of event 3 : ", type(self.event))
#action = {"action": "RotateLeft", 'rotation':90}
#action = "RotateLook, rotation=-90"
#self.event = self.env.step(action)
#print ("type of event 4 : ", type(self.event))
#self.event = self.event.events[0]
#mcs_output = wrap_output(self.event)
#lastActionSuccess = self.event.metadata['lastActionSuccess']
lastActionSuccess = self.event.return_status
print("last action sucess", lastActionSuccess)
self.process_frame()
#print (self.get_pose())
#self.pose = game_util.get_pose(self.event)
print("current pose = ", self.pose)
self.board = None
#point_dists = np.sum(np.abs(self.graph.points - np.array(self.end_point[:2])), axis=1)
#print ("jsut b4 end of function")
#dist_min = np.min(point_dists)
#self.is_possible_end_point = int(dist_min < 0.0001)
print("end of reset in game state function")
def reset(self, seed=None, test_ind=None):
if seed is not None:
self.local_random.seed(seed)
question_row, question_type_ind = test_ind
question_type = self.question_types[question_type_ind]
question_data = self.test_datasets[question_type_ind][
question_row % len(self.test_datasets[question_type_ind])]
scene_num, scene_seed, question_str, answer = question_data[1:5]
self.scene_seed = int(scene_seed)
self.scene_num = int(scene_num)
self.question_str = question_str
self.question_type_ind = question_type_ind
self.scene_name = 'FloorPlan%d' % self.scene_num
grid_file = 'layouts/%s-layout.npy' % self.scene_name
self.points = (np.load(grid_file) * 1.0 /
constants.AGENT_STEP_SIZE).astype(int)
max_num_repeats = 1
remove_prob = 0.5
if question_type == 'existence':
max_num_repeats = 10
remove_prob = 0.25
elif question_type == 'counting':
max_num_repeats = constants.MAX_COUNTING_ANSWER + 1
remove_prob = 0.5
elif question_type == 'contains':
max_num_repeats = 10
remove_prob = 0.25
self.event = game_util.reset(self.env,
self.scene_name,
render_image=True,
render_depth_image=True,
render_class_image=True,
render_object_image=True)
self.agent_height = self.event.metadata['agent']['position']['y']
self.event = self.env.random_initialize(
self.scene_seed,
max_num_repeats=max_num_repeats,
remove_prob=remove_prob)
print('Question: %s' % self.question_str)
if answer == 'True':
self.answer = True
elif answer == 'False':
self.answer = True
else:
self.answer = int(answer)
start_point = self.local_random.randint(0, self.points.shape[0] - 1)
start_point = self.points[start_point, :].copy()
self.start_point = (start_point[0], start_point[1],
self.local_random.randint(0, 3))
action = {
'action': 'TeleportFull',
'x': self.start_point[0] * constants.AGENT_STEP_SIZE,
'y': self.agent_height,
'z': self.start_point[1] * constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'rotation': self.start_point[2] * 90,
'horizon': 30,
}
self.event = self.env.step(action)
self.process_frame()
def reset(self, scene_name=None, use_gt=True, seed=None):
if scene_name is None:
# Do half reset
action_ind = self.local_random.randint(
0, constants.STEPS_AHEAD**2 - 1)
action_x = action_ind % constants.STEPS_AHEAD - int(
constants.STEPS_AHEAD / 2)
action_z = int(action_ind / constants.STEPS_AHEAD) + 1
x_shift = 0
z_shift = 0
if self.pose[2] == 0:
x_shift = action_x
z_shift = action_z
elif self.pose[2] == 1:
x_shift = action_z
z_shift = -action_x
elif self.pose[2] == 2:
x_shift = -action_x
z_shift = -action_z
elif self.pose[2] == 3:
x_shift = -action_z
z_shift = action_x
action_x = self.pose[0] + x_shift
action_z = self.pose[1] + z_shift
self.end_point = (action_x, action_z, self.pose[2])
else:
# Do full reset
self.scene_name = scene_name
grid_file = 'layouts/%s-layout.npy' % scene_name
self.graph = graph_obj.Graph(grid_file, use_gt=use_gt)
if seed is not None:
self.local_random.seed(seed)
lastActionSuccess = False
self.bounds = [
self.graph.xMin, self.graph.yMin,
self.graph.xMax - self.graph.xMin + 1,
self.graph.yMax - self.graph.yMin + 1
]
while not lastActionSuccess:
self.event = game_util.reset(self.env, self.scene_name)
self.agent_height = self.event.metadata['agent']['position'][
'y']
self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET
self.event = self.env.random_initialize(seed)
start_point = self.local_random.randint(
0, self.graph.points.shape[0] - 1)
start_point = self.graph.points[start_point, :].copy()
self.start_point = (start_point[0], start_point[1],
self.local_random.randint(0, 3))
self.end_point = self.start_point
while self.end_point[0] == self.start_point[
0] and self.end_point[1] == self.start_point[1]:
end_point = self.local_random.randint(
0, self.graph.points.shape[0] - 1)
end_point = self.graph.points[end_point, :].copy()
self.end_point = [
end_point[0], end_point[1],
self.local_random.randint(0, 3)
]
self.end_point[0] += self.local_random.randint(
-constants.TERMINAL_CHECK_PADDING,
constants.TERMINAL_CHECK_PADDING)
self.end_point[1] += self.local_random.randint(
-constants.TERMINAL_CHECK_PADDING,
constants.TERMINAL_CHECK_PADDING)
self.end_point = tuple(self.end_point)
action = {
'action': 'TeleportFull',
'x': self.start_point[0] * constants.AGENT_STEP_SIZE,
'y': self.agent_height,
'z': self.start_point[1] * constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'rotation': self.start_point[2] * 90,
'horizon': 60
}
self.event = self.env.step(action)
lastActionSuccess = self.event.metadata['lastActionSuccess']
self.process_frame()
self.board = None
point_dists = np.sum(np.abs(self.graph.points -
np.array(self.end_point[:2])),
axis=1)
dist_min = np.min(point_dists)
self.is_possible_end_point = int(dist_min < 0.0001)
def get_obj(env, open_test_objs, reachable_points, agent_height, scene_name, good_obj_point):
# Reset the scene to put all the objects back where they started.
game_util.reset(env, scene_name,
render_image=False,
render_depth_image=False,
render_class_image=False,
render_object_image=True)
if good_obj_point is not None:
search_points = {good_obj_point[0]}
else:
search_points = reachable_points
for point in search_points:
for rotation in range(4):
if good_obj_point is not None and rotation != good_obj_point[1]:
continue
for horizon in [-30, 0, 30]:
if good_obj_point is not None and horizon != good_obj_point[2]:
continue
action = {'action': 'TeleportFull',
'x': point[0],
'y': agent_height,
'z': point[1],
'rotateOnTeleport': True,
'rotation': rotation * 90,
'horizon': horizon
}
event = env.step(action)
if event.metadata['lastActionSuccess']:
# Close everything.
for obj in event.metadata['objects']:
if (obj['visible'] and obj['objectId'] and obj['isOpen'] and
obj['objectType'] in constants.VAL_RECEPTACLE_OBJECTS):
action = {'action': 'CloseObject',
'objectId': obj['objectId']}
event = env.step(action)
# Try to pick up a valid inv object.
for obj in event.metadata['objects']:
if obj['visible'] and obj['objectType'] in open_test_objs:
action = {'action': 'PickupObject',
'objectId': obj['objectId']}
event = env.step(action)
if event.metadata['lastActionSuccess']:
good_obj_point = (point, rotation, horizon)
return good_obj_point
# Open everything.
for open_obj in event.metadata['objects']:
if (open_obj['visible'] and open_obj['objectId'] and open_obj['openable'] and
not open_obj['pickupable'] and
open_obj['objectType'] in constants.VAL_RECEPTACLE_OBJECTS):
action = {'action': 'OpenObject',
'objectId': open_obj['objectId']}
event = env.step(action)
if event.metadata['lastActionSuccess']:
# Try to pick up a valid inv object.
for obj in event.metadata['objects']:
if obj['visible'] and obj['objectType'] in open_test_objs:
action = {'action': 'PickupObject',
'objectId': obj['objectId']}
event = env.step(action)
if event.metadata['lastActionSuccess']:
good_obj_point = (point, rotation, horizon)
return good_obj_point
# Close open_obj.
action = {'action': 'CloseObject',
'objectId': open_obj['objectId']}
event = env.step(action)
return None
def run():
print(all_scene_numbers)
# create env and agent
env = game_util.create_env(build_path=constants.BUILD_PATH,
quality='Low')
while len(all_scene_numbers) > 0:
lock.acquire()
scene_num = all_scene_numbers.pop()
lock.release()
fn = os.path.join('layouts', ('FloorPlan%d-layout.npy') % scene_num)
if os.path.isfile(fn):
print("file %s already exists; skipping this floorplan" % fn)
continue
openable_json_file = os.path.join('layouts', ('FloorPlan%d-openable.json') % scene_num)
scene_objs_json_file = os.path.join('layouts', ('FloorPlan%d-objects.json') % scene_num)
scene_name = ('FloorPlan%d') % scene_num
print('Running ' + scene_name)
event = game_util.reset(env, scene_name,
render_image=False,
render_depth_image=False,
render_class_image=False,
render_object_image=True)
agent_height = event.metadata['agent']['position']['y']
scene_objs = list(set([obj['objectType'] for obj in event.metadata['objects']]))
with open(scene_objs_json_file, 'w') as sof:
json.dump(scene_objs, sof, sort_keys=True, indent=4)
# Get all the reachable points through Unity for this step size.
event = env.step(dict(action='GetReachablePositions',
gridSize=constants.AGENT_STEP_SIZE / constants.RECORD_SMOOTHING_FACTOR))
if event.metadata['actionReturn'] is None:
print("ERROR: scene %d 'GetReachablePositions' returns None" % scene_num)
else:
reachable_points = set()
for point in event.metadata['actionReturn']:
reachable_points.add((point['x'], point['z']))
print("scene %d got %d reachable points, now checking" % (scene_num, len(reachable_points)))
# Pick up a small object to use in testing whether points are good for openable objects.
open_test_objs = {'ButterKnife', 'CD', 'CellPhone', 'Cloth', 'CreditCard', 'DishSponge', 'Fork',
'KeyChain', 'Pen', 'Pencil', 'SoapBar', 'Spoon', 'Watch'}
good_obj_point = None
good_obj_point = get_obj(env, open_test_objs, reachable_points, agent_height, scene_name, good_obj_point)
best_open_point = {} # map from object names to the best point from which they can be successfully opened
best_sem_coverage = {} # number of pixels in the semantic map of the receptacle at the existing best openpt
checked_points = set()
scene_receptacles = set()
for point in reachable_points:
point_is_valid = True
action = {'action': 'TeleportFull',
'x': point[0],
'y': agent_height,
'z': point[1],
}
event = env.step(action)
if event.metadata['lastActionSuccess']:
for horizon in [-30, 0, 30]:
action = {'action': 'TeleportFull',
'x': point[0],
'y': agent_height,
'z': point[1],
'rotateOnTeleport': True,
'rotation': 0,
'horizon': horizon
}
event = env.step(action)
if not event.metadata['lastActionSuccess']:
point_is_valid = False
break
for rotation in range(3):
action = {'action': 'RotateLeft'}
event = env.step(action)
if not event.metadata['lastActionSuccess']:
point_is_valid = False
break
if not point_is_valid:
break
if point_is_valid:
checked_points.add(point)
else:
continue
# Check whether we can open objects from here in any direction with any tilt.
for rotation in range(4):
# First try up, then down, then return to the horizon before moving again.
for horizon in [-30, 0, 30]:
action = {'action': 'TeleportFull',
'x': point[0],
'y': agent_height,
'z': point[1],
'rotateOnTeleport': True,
'rotation': rotation * 90,
'horizon': horizon
}
event = env.step(action)
for obj in event.metadata['objects']:
if (obj['visible'] and obj['objectId'] and obj['receptacle'] and not obj['pickupable']
and obj['objectType'] in constants.VAL_RECEPTACLE_OBJECTS):
obj_name = obj['objectId']
obj_point = (obj['position']['x'], obj['position']['y'])
scene_receptacles.add(obj_name)
# Go ahead and attempt to close the object from this position if it's open.
if obj['openable'] and obj['isOpen']:
close_action = {'action': 'CloseObject',
'objectId': obj['objectId']}
event = env.step(close_action)
point_to_recep = np.linalg.norm(np.array(point) - np.array(obj_point))
if len(env.last_event.metadata['inventoryObjects']) > 0:
inv_obj = env.last_event.metadata['inventoryObjects'][0]['objectId']
else:
inv_obj = None
# Heuristic implemented in task_game_state has agent 0.5 or farther in agent space.
heuristic_far_enough_from_recep = 0.5 < point_to_recep
# Ensure this point affords a larger view according to the semantic segmentation
# of the receptacle than the existing.
point_sem_coverage = get_mask_of_obj(env, obj['objectId'])
if point_sem_coverage is None:
use_sem_heuristic = False
better_sem_covereage = False
else:
use_sem_heuristic = True
better_sem_covereage = (obj_name not in best_sem_coverage or
best_sem_coverage[obj_name] is None or
point_sem_coverage > best_sem_coverage[obj_name])
# Ensure that this point is farther away than our existing best candidate.
# We'd like to open each receptacle from as far away as possible while retaining
# the ability to pick/place from it.
farther_than_existing_good_point = (obj_name not in best_open_point or
point_to_recep >
np.linalg.norm(
np.array(point) -
np.array(best_open_point[obj_name][:2])))
# If we don't have an inventory object, though, we'll fall back to the heuristic
# of being able to open/close as _close_ as possible.
closer_than_existing_good_point = (obj_name not in best_open_point or
point_to_recep <
np.linalg.norm(
np.array(point) -
np.array(best_open_point[obj_name][:2])))
# Semantic segmentation heuristic.
if ((use_sem_heuristic and heuristic_far_enough_from_recep and better_sem_covereage)
or (not use_sem_heuristic and
# Distance heuristics.
(heuristic_far_enough_from_recep and
(inv_obj and farther_than_existing_good_point) or
(not inv_obj and closer_than_existing_good_point)))):
if obj['openable']:
action = {'action': 'OpenObject',
'objectId': obj['objectId']}
event = env.step(action)
if not obj['openable'] or event.metadata['lastActionSuccess']:
# We can open the object, so try placing our small inventory obj inside.
# If it can be placed inside and retrieved, then this is a safe point.
action = {'action': 'PutObject',
'objectId': inv_obj,
'receptacleObjectId': obj['objectId'],
'forceAction': True,
'placeStationary': True}
if inv_obj:
event = env.step(action)
if inv_obj is None or event.metadata['lastActionSuccess']:
action = {'action': 'PickupObject',
'objectId': inv_obj}
if inv_obj:
event = env.step(action)
if inv_obj is None or event.metadata['lastActionSuccess']:
# Finally, ensure we can also close the receptacle.
if obj['openable']:
action = {'action': 'CloseObject',
'objectId': obj['objectId']}
event = env.step(action)
if not obj['openable'] or event.metadata['lastActionSuccess']:
# We can put/pick our inv object into the receptacle from here.
# We have already ensured this point is farther than any
# existing best, so this is the new best.
best_open_point[obj_name] = [point[0], point[1], rotation * 90, horizon]
best_sem_coverage[obj_name] = point_sem_coverage
# We could not retrieve our inv object, so we need to go get another one
else:
good_obj_point = get_obj(env, open_test_objs, reachable_points,
agent_height, scene_name, good_obj_point)
action = {'action': 'TeleportFull',
'x': point[0],
'y': agent_height,
'z': point[1],
'rotateOnTeleport': True,
'rotation': rotation * 90,
'horizon': horizon
}
event = env.step(action)
# Regardless of what happened up there, try to close the receptacle again if
# it remained open.
if obj['isOpen']:
action = {'action': 'CloseObject',
'objectId': obj['objectId']}
event = env.step(action)
essential_objs = []
if scene_num in constants.SCENE_TYPE["Kitchen"]:
essential_objs.extend(["Microwave", "Fridge"])
for obj in essential_objs:
if not np.any([obj in obj_key for obj_key in best_open_point]):
print("WARNING: Essential object %s has no open points in scene %d" % (obj, scene_num))
print("scene %d found open/pick/place/close positions for %d/%d receptacle objects" %
(scene_num, len(best_open_point), len(scene_receptacles)))
with open(openable_json_file, 'w') as f:
json.dump(best_open_point, f, sort_keys=True, indent=4)
print("scene %d reachable %d, checked %d; taking intersection" %
(scene_num, len(reachable_points), len(checked_points)))
points = np.array(list(checked_points))[:, :2]
points = points[np.lexsort((points[:, 0], points[:, 1])), :]
np.save(fn, points)
env.stop()
print('Done')
def reset(self, scene_name=None, use_gt=True, seed=None, config_filename= "",event=None):
if scene_name is None:
# Do half reset
action_ind = self.local_random.randint(0, constants.STEPS_AHEAD ** 2 - 1)
action_x = action_ind % constants.STEPS_AHEAD - int(constants.STEPS_AHEAD / 2)
action_z = int(action_ind / constants.STEPS_AHEAD) + 1
x_shift = 0
z_shift = 0
if self.pose[2] == 0:
x_shift = action_x
z_shift = action_z
elif self.pose[2] == 1:
x_shift = action_z
z_shift = -action_x
elif self.pose[2] == 2:
x_shift = -action_x
z_shift = -action_z
elif self.pose[2] == 3:
x_shift = -action_z
z_shift = action_x
action_x = self.pose[0] + x_shift
action_z = self.pose[1] + z_shift
self.end_point = (action_x, action_z, self.pose[2])
#print ("in the game state reset end point is : ", self.end_point)
else:
# Do full reset
#self.world_poly = fov.FieldOfView([0, 0, 0], 0, [])
self.world_poly = sp.Polygon()
self.goals_found = False
self.scene_name = scene_name
self.number_actions = 0
self.id_goal_in_hand = None
#print ("Full reset - in the first time of load")
#grid_file = 'layouts/%s-layout_%s.npy' % (scene_name,str(constants.AGENT_STEP_SIZE))
self.graph = None
self.goal_in_hand = False
if seed is not None:
self.local_random.seed(seed)
lastActionSuccess = False
self.discovered_explored = {}
self.discovered_objects = []
self.bounds = None
#while True :
#self.event = self.event.events[0]
if event != None :
self.event = event
else :
self.event = game_util.reset(self.env, self.scene_name,config_filename)
self.goals = []
for key,value in self.event.goal.metadata.items():
if key == "target" or key == "target_1" or key == "target_2":
self.goals.append(self.event.goal.metadata[key]["id"])
for obj in self.event.object_list:
if obj.uuid not in self.discovered_explored:
print("uuid : ", obj.uuid)
self.discovered_explored[obj.uuid] = {0: obj.position}
self.discovered_objects.append(obj.__dict__)
self.discovered_objects[-1]['locationParent'] = None
self.discovered_objects[-1]['explored'] = 0
self.discovered_objects[-1]['openable'] = None
#self.discovered_objects[-1]['agent_position'] = None
self.add_obstacle_func(self.event)
#print ("type of event 2 : ", type(self.event))
lastActionSuccess = self.event.return_status
#break
self.process_frame()
self.board = None
