def load_nav_graph(self):
'''
build navigation grid graph
'''
floor_plan = self.traj['scene']['floor_plan']
scene_num = self.traj['scene']['scene_num']
self.gt_graph = graph_obj.Graph(use_gt=True, construct_graph=True, scene_id=scene_num)
def reset(self, scene_name=None, seed=None):
if scene_name is not None:
if self.game_state.env is not None and type(
self.game_state) == GameState:
self.game_state.reset(scene_name, use_gt=False, seed=seed)
self.gt_graph = graph_obj.Graph('layouts/%s-layout.npy' %
scene_name,
use_gt=True)
self.bounds = [
self.game_state.graph.xMin, self.game_state.graph.yMin,
self.game_state.graph.xMax - self.game_state.graph.xMin + 1,
self.game_state.graph.yMax - self.game_state.graph.yMin + 1
]
if len(self.game_state.end_point) == 0:
self.game_state.end_point = (self.game_state.graph.xMin +
constants.TERMINAL_CHECK_PADDING,
self.game_state.graph.yMin +
constants.TERMINAL_CHECK_PADDING,
0)
self.action = np.zeros(self.action_util.num_actions)
self.memory = np.zeros(
(constants.SPATIAL_MAP_HEIGHT, constants.SPATIAL_MAP_WIDTH,
constants.MEMORY_SIZE))
self.gru_state = np.zeros((1, constants.GRU_SIZE))
self.pose = self.game_state.pose
self.is_possible = 1
self.num_steps = 0
self.times = np.zeros(2)
self.impossible_spots = set()
self.visited_spots = set()
else:
self.game_state.reset()
self.goal_pose = np.array([
self.game_state.end_point[0] - self.game_state.graph.xMin,
self.game_state.end_point[1] - self.game_state.graph.yMin
],
dtype=np.int32)[:2]
self.inference()
def reset(self, scene_name=None, seed=None, config_filename=""):
if scene_name is not None:
if self.game_state.env is not None and type(
self.game_state) == GameState:
self.game_state.reset(scene_name,
use_gt=False,
seed=seed,
config_filename=config_filename)
self.gt_graph = graph_obj.Graph(
'layouts/%s-layout_%s.npy' %
(scene_name, str(constants.AGENT_STEP_SIZE)),
self.action_util,
use_gt=True)
self.bounds = [
self.game_state.graph.xMin, self.game_state.graph.yMin,
self.game_state.graph.xMax - self.game_state.graph.xMin + 1,
self.game_state.graph.yMax - self.game_state.graph.yMin + 1
]
'''
if len(self.game_state.end_point) == 0:
self.game_state.end_point = (self.game_state.graph.xMin + constants.TERMINAL_CHECK_PADDING,
self.game_state.graph.yMin + constants.TERMINAL_CHECK_PADDING, 0)
print ("in agent reset end point set to : " , self.game_state.end_point)
'''
self.action = np.zeros(self.action_util.num_actions)
self.memory = np.zeros(
(constants.SPATIAL_MAP_HEIGHT, constants.SPATIAL_MAP_WIDTH,
constants.MEMORY_SIZE))
self.gru_state = np.zeros((1, constants.GRU_SIZE))
self.pose = self.game_state.pose
self.is_possible = 1
self.num_steps = 0
self.times = np.zeros(2)
self.impossible_spots = set()
self.visited_spots = set()
else:
#print ("")
self.game_state.reset()
def create_dump():
time_str = py_util.get_time_str()
prefix = 'questions/'
if not os.path.exists(prefix + dataset_type + '/data_contains'):
os.makedirs(prefix + dataset_type + '/data_contains')
h5 = h5py.File(prefix + dataset_type + '/data_contains/Contains_Questions_' + time_str + '.h5', 'w')
h5.create_dataset('questions/question', (num_record, 5), dtype=np.int32)
print('Generating %d contains questions' % num_record)
# Generate contains questions
data_ind = 0
episode = Episode()
scene_number = -1
while data_ind < num_record:
k = 0
scene_number += 1
scene_num = scene_numbers[scene_number % len(scene_numbers)]
scene_name = 'FloorPlan%d' % scene_num
episode.initialize_scene(scene_name)
num_tries = 0
while k < num_samples_per_scene and num_tries < 10 * num_samples_per_scene:
# randomly pick a pickable object in the scene
object_class = random.choice(all_object_classes)
generated_no, generated_yes = False, False
question = None
temp_data = []
num_tries += 1
grid_file = 'layouts/%s-layout.npy' % scene_name
xray_graph = graph_obj.Graph(grid_file, use_gt=True, construct_graph=False)
scene_bounds = [xray_graph.xMin, xray_graph.yMin,
xray_graph.xMax - xray_graph.xMin + 1,
xray_graph.yMax - xray_graph.yMin + 1]
for i in range(20): # try 20 times
scene_seed = random.randint(0, 999999999)
episode.initialize_episode(scene_seed=scene_seed) # randomly initialize the scene
if not question:
question = ExistenceQuestion.get_true_contain_question(episode, all_object_classes, receptacles)
if DEBUG:
print(str(question))
if not question:
continue
object_class = question.object_class
parent_class = question.parent_object_class
answer = question.get_answer(episode)
object_target = constants.OBJECT_CLASS_TO_ID[object_class]
if answer and not generated_yes:
event = episode.event
xray_graph.memory[:, :, 1:] = 0
objs = {obj['objectId']: obj for obj in event.metadata['objects']
if obj['objectType'] == object_class and obj['parentReceptacle'].split('|')[0] == parent_class}
for obj in objs.values():
if obj['objectType'] != object_class:
continue
obj_point = game_util.get_object_point(obj, scene_bounds)
xray_graph.memory[obj_point[1], obj_point[0],
constants.OBJECT_CLASS_TO_ID[obj['objectType']] + 1] = 1
# Make sure findable
try:
graph_points = xray_graph.points.copy()
graph_points = graph_points[np.random.permutation(graph_points.shape[0]), :]
num_checked_points = 0
for start_point in graph_points:
headings = np.random.permutation(4)
for heading in headings:
start_point = (start_point[0], start_point[1], heading)
patch = xray_graph.get_graph_patch(start_point)[0]
if patch[:, :, object_target + 1].max() > 0:
action = {'action': 'TeleportFull',
'x': start_point[0] * constants.AGENT_STEP_SIZE,
'y': episode.agent_height,
'z': start_point[1] * constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'rotation': start_point[2] * 90,
'horizon': -30,
}
event = episode.env.step(action)
num_checked_points += 1
if num_checked_points > 1000:
answer = None
raise AssertionError
for jj in range(4):
open_success = True
opened_objects = set()
parents = [game_util.get_object(obj['parentReceptacle'], event.metadata)
for obj in objs.values()]
openable_parents = [parent for parent in parents
if parent['visible'] and parent['openable'] and not parent['isopen']]
while open_success:
for obj in objs.values():
if obj['objectId'] in event.instance_detections2D:
if game_util.check_object_size(event.instance_detections2D[obj['objectId']]):
raise AssertionError
if len(openable_parents) > 0:
action = {'action': 'OpenObject'}
game_util.set_open_close_object(action, event)
event = episode.env.step(action)
open_success = event.metadata['lastActionSuccess']
if open_success:
opened_objects.add(episode.env.last_action['objectId'])
else:
open_success = False
for opened in opened_objects:
event = episode.env.step({
'action': 'CloseObject',
'objectId': opened,
'forceVisible': True})
if not event.metadata['lastActionSuccess']:
answer = None
raise AssertionError
if jj < 3:
event = episode.env.step({'action': 'LookDown'})
answer = None
except AssertionError:
if answer is None and DEBUG:
print('failed to generate')
print(str(question), answer)
if answer == False and not generated_no:
generated_no = True
temp_data.append([scene_num, scene_seed, constants.OBJECT_CLASS_TO_ID[object_class], constants.OBJECT_CLASS_TO_ID[parent_class], answer])
elif answer == True and not generated_yes:
generated_yes = True
temp_data.append([scene_num, scene_seed, constants.OBJECT_CLASS_TO_ID[object_class], constants.OBJECT_CLASS_TO_ID[parent_class], answer])
if generated_no and generated_yes:
h5['questions/question'][data_ind, :] = np.array(temp_data[0])
h5['questions/question'][data_ind + 1, :] = np.array(temp_data[1])
h5.flush()
data_ind += 2
k += 2
break
print("# generated samples: {}".format(data_ind))
h5.close()
episode.env.stop_unity()
def reset(self, seed=None, test_ind=None):
self.board = None
self.seen_object = False
self.terminal = False
self.opened_receptacles = set()
self.closed_receptacles = set()
self.seen_obj1 = set()
self.seen_obj2 = set()
self.visited_locations = set()
self.can_end = False
if seed is not None:
self.local_random.seed(seed)
# Do equal number of each question type in train.
question_type_ind = self.local_random.sample(
constants.USED_QUESTION_TYPES, 1)[0]
# get random row
if test_ind is not None:
question_row, question_type_ind = test_ind
question_type = self.question_types[question_type_ind]
question_data = self.test_datasets[question_type_ind][
question_row, :]
test_ind = (question_row, question_type_ind)
else:
question_type = self.question_types[question_type_ind]
question_row = self.local_random.randint(
0,
len(self.datasets[question_type_ind]) - 1)
question_data = self.datasets[question_type_ind][question_row, :]
container_ind = None
if question_type_ind == 0 or question_type_ind == 1:
scene_num, scene_seed, object_ind, answer = question_data
self.question_target = object_ind
if question_type_ind == 0:
answer = bool(answer)
elif question_type_ind == 2:
scene_num, scene_seed, object_ind, container_ind, answer = question_data
answer = bool(answer)
self.question_target = (object_ind, container_ind)
else:
raise Exception('No question type found for type %d' %
question_type_ind)
self.scene_seed = scene_seed
self.scene_num = scene_num
self.object_target = object_ind
self.parent_target = container_ind
self.container_target = np.zeros(constants.NUM_CLASSES)
self.direction_target = np.zeros(4)
if container_ind is not None:
self.container_target[container_ind] = 1
self.question_type_ind = question_type_ind
self.scene_name = 'FloorPlan%d' % scene_num
grid_file = 'layouts/%s-layout.npy' % self.scene_name
self.graph = graph_obj.Graph(grid_file, use_gt=False)
self.xray_graph = graph_obj.Graph(grid_file, use_gt=True)
self.bounds = [
self.graph.xMin, self.graph.yMin,
self.graph.xMax - self.graph.xMin + 1,
self.graph.yMax - self.graph.yMin + 1
]
max_num_repeats = 1
remove_prob = 0.5
if question_type == 'existence':
max_num_repeats = 10
remove_prob = 0.25
elif question_type == 'counting':
max_num_repeats = constants.MAX_COUNTING_ANSWER + 1
remove_prob = 0.5
elif question_type == 'contains':
max_num_repeats = 10
remove_prob = 0.25
self.event = game_util.reset(self.env, self.scene_name)
self.agent_height = self.event.metadata['agent']['position']['y']
self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET
self.event = self.env.random_initialize(
self.scene_seed,
max_num_repeats=max_num_repeats,
remove_prob=remove_prob)
print('Type:', question_type, 'Row: ', question_row, 'Scene',
self.scene_name, 'seed', scene_seed)
print(
'Question:',
game_util.get_question_str(question_type_ind, object_ind,
container_ind))
if self.question_type_ind == 2:
print('Answer:', constants.OBJECTS[object_ind], 'in',
constants.OBJECTS[container_ind], 'is', answer)
else:
print('Answer:', constants.OBJECTS[object_ind], 'is', answer)
self.answer = answer
# Verify answer
if self.question_type_ind == 0:
objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind],
self.event.metadata)
computed_answer = len(objs) > 0
requires_interaction = True
for obj in objs:
parent = obj['parentReceptacle'].split('|')[0]
if parent not in {'Fridge', 'Cabinet', 'Microwave'}:
requires_interaction = False
break
elif self.question_type_ind == 1:
objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind],
self.event.metadata)
computed_answer = len(objs)
requires_interaction = True
elif self.question_type_ind == 2:
objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind],
self.event.metadata)
if len(objs) == 0:
computed_answer = False
requires_interaction = constants.OBJECTS[
self.question_target[1]] in {
'Fridge', 'Cabinet', 'Microwave'
}
else:
obj = objs[0]
computed_answer = False
for obj in objs:
requires_interaction = True
parent = obj['parentReceptacle'].split('|')[0]
if parent in constants.OBJECT_CLASS_TO_ID:
parent_ind = constants.OBJECT_CLASS_TO_ID[parent]
computed_answer = parent_ind == self.question_target[1]
if computed_answer:
if parent not in {
'Fridge', 'Cabinet', 'Microwave'
}:
requires_interaction = False
break
else:
computed_answer = False
self.requires_interaction = requires_interaction
try:
assert self.answer == computed_answer, 'Answer does not match scene metadata'
except AssertionError:
print('Type:', question_type, 'Row: ', question_row, 'Scene',
self.scene_name, 'seed', scene_seed)
print('Answer', computed_answer, 'does not match expected value',
self.answer, ', did randomization process change?')
pdb.set_trace()
self.answer = computed_answer
if constants.NUM_CLASSES > 1:
self.hidden_items = set()
objects = self.event.metadata['objects']
for obj in objects:
if obj['receptacle'] and obj['openable'] and not obj['isopen']:
for inside_obj in obj['receptacleObjectIds']:
self.hidden_items.add(inside_obj)
objects = self.event.metadata['objects']
for obj in objects:
if obj['objectType'] not in constants.OBJECT_CLASS_TO_ID:
continue
obj_bounds = game_util.get_object_bounds(obj, self.bounds)
self.xray_graph.memory[
obj_bounds[1]:obj_bounds[3], obj_bounds[0]:obj_bounds[2],
constants.OBJECT_CLASS_TO_ID[obj['objectType']] + 1] = 1
start_point = self.local_random.randint(0,
self.graph.points.shape[0] - 1)
start_point = self.graph.points[start_point, :].copy()
self.start_point = (start_point[0], start_point[1],
self.local_random.randint(0, 3))
action = {
'action': 'TeleportFull',
'x': self.start_point[0] * constants.AGENT_STEP_SIZE,
'y': self.agent_height,
'z': self.start_point[1] * constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'rotation': self.start_point[2] * 90,
'horizon': 30,
}
self.event = self.env.step(action)
self.process_frame()
self.reward = 0
self.end_point = []
def reset(self,
scene_name=None,
use_gt=True,
seed=None,
config_filename=""):
if scene_name is None:
# Do half reset
action_ind = self.local_random.randint(
0, constants.STEPS_AHEAD**2 - 1)
action_x = action_ind % constants.STEPS_AHEAD - int(
constants.STEPS_AHEAD / 2)
action_z = int(action_ind / constants.STEPS_AHEAD) + 1
x_shift = 0
z_shift = 0
if self.pose[2] == 0:
x_shift = action_x
z_shift = action_z
elif self.pose[2] == 1:
x_shift = action_z
z_shift = -action_x
elif self.pose[2] == 2:
x_shift = -action_x
z_shift = -action_z
elif self.pose[2] == 3:
x_shift = -action_z
z_shift = action_x
action_x = self.pose[0] + x_shift
action_z = self.pose[1] + z_shift
self.end_point = (action_x, action_z, self.pose[2])
#print ("in the game state reset end point is : ", self.end_point)
else:
# Do full reset
self.scene_name = scene_name
#print ("Full reset - in the first time of load")
grid_file = 'layouts/%s-layout_%s.npy' % (
scene_name, str(constants.AGENT_STEP_SIZE))
self.graph = graph_obj.Graph(grid_file,
self.action_util,
use_gt=use_gt)
if seed is not None:
self.local_random.seed(seed)
lastActionSuccess = False
self.discovered_explored = {}
self.discovered_objects = []
self.bounds = [
self.graph.xMin, self.graph.yMin,
self.graph.xMax - self.graph.xMin + 1,
self.graph.yMax - self.graph.yMin + 1
]
'''
while not lastActionSuccess:
print ("In the while loop")
self.event = game_util.reset(self.env, self.scene_name)
self.event = self.event.events[0]
self.agent_height = self.event.metadata['agent']['position']['y']
self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET
#self.event = self.env.random_initialize(seed)
#self.start_point = []
start_point_2 = []
#start_point_2.append(int(self.event.metadata['agent']['position']['x']/constants.AGENT_STEP_SIZE) -self.graph.xMin +1)
#start_point_2.append(int(self.event.metadata['agent']['position']['z']/constants.AGENT_STEP_SIZE) - self.graph.yMin +1 )
start_point_2.append(int(self.event.metadata['agent']['position']['x']/constants.AGENT_STEP_SIZE) )
start_point_2.append(int(self.event.metadata['agent']['position']['z']/constants.AGENT_STEP_SIZE) )
start_point_2.append(int(self.event.metadata['agent']['rotation']['y']/90))
print(start_point_2)
print ("starting horizon", self.event.metadata['agent']['cameraHorizon'])
self.start_point = start_point_2[:]
#start_point = self.local_random.randint(0, self.graph.points.shape[0] - 1)
#start_point = self.graph.points[start_point, :].copy()
#self.start_point = (start_point[0], start_point[1], self.local_random.randint(0, 3))
self.end_point = self.start_point
#print ("B4 the while loop for assigning end point", self.start_point)
while self.end_point[0] == self.start_point[0] and self.end_point[1] == self.start_point[1]:
#end_point = self.local_random.randint(0, self.graph.points.shape[0] - 1)
#end_point = self.graph.points[end_point, :].copy()
#self.end_point = [end_point[0], end_point[1], self.local_random.randint(0, 3)]
#self.end_point[0] += self.local_random.randint(-constants.TERMINAL_CHECK_PADDING, constants.TERMINAL_CHECK_PADDING)
#self.end_point[1] += self.local_random.randint(-constants.TERMINAL_CHECK_PADDING, constants.TERMINAL_CHECK_PADDING)
#self.end_point = tuple(self.end_point)
self.end_point = (self.end_point[0]-1 , self.end_point[1]-1 , self.end_point[2])
print ("In the while loop for assigning end point",self.end_point)
action = {'action': 'TeleportFull',
'x': self.start_point[0] * constants.AGENT_STEP_SIZE,
'y': self.agent_height,
'z': self.start_point[1] * constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'rotation': self.start_point[2] * 90,
'horizon': 0}
#'horizon': 60}
'''
while True:
self.event = game_util.reset(self.env, self.scene_name,
config_filename)
#self.event = self.event.events[0]
print("type of event 2 : ", type(self.event))
lastActionSuccess = self.event.return_status
break
start_point_2 = []
'''
self.agent_height = self.event.metadata['agent']['position']['y']
self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET
start_point_2.append(int(self.event.metadata['agent']['position']['x']/constants.AGENT_STEP_SIZE) )
start_point_2.append(int(self.event.metadata['agent']['position']['z']/constants.AGENT_STEP_SIZE) )
start_point_2.append(int(self.event.metadata['agent']['rotation']['y']/90))
'''
print("rotation ", self.event.rotation)
rotation_angle = -(self.event.rotation % 90)
action = "RotateLook, rotation=%d" % int(rotation_angle)
self.goal = self.event.goal
#action = {'action':"Pass"}
#print ("z before turning ", self.event.position['z']/constants.AGENT_STEP_SIZE)
#print ("z before turning ", self.event.position['z'])
self.event = self.env.step(action)
#print ("z after turning ", self.event.position['z']/constants.AGENT_STEP_SIZE)
#print ("z after turning ", self.event.position['z'])
self.agent_height = self.event.position['y']
self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET
start_point_2.append(
math.floor(self.event.position['x'] /
constants.AGENT_STEP_SIZE))
start_point_2.append(
math.floor(self.event.position['z'] /
constants.AGENT_STEP_SIZE))
start_point_2.append(int(self.event.rotation / 90))
#print(start_point_2)
self.start_point = start_point_2[:]
self.end_point = self.start_point[:]
self.end_point = (self.end_point[0], self.end_point[1],
(self.end_point[2] + 1) % 4)
#print ("end point = ", self.end_point)
#print ("start point = ", self.start_point)
#print ("last action success", self.event['lastActionSuccess'])
#print (self.event)
#print ("last action success", self.event.lastActionSuccess)
#print ("number of objects : ",len(self.event.metadata['objects']))
#action = {"action": "RotateRight", 'rotatio#n':90}
#print ("type of event 3 : ", type(self.event))
#action = {"action": "RotateLeft", 'rotation':90}
#action = "RotateLook, rotation=-90"
#self.event = self.env.step(action)
#print ("type of event 4 : ", type(self.event))
#self.event = self.event.events[0]
#mcs_output = wrap_output(self.event)
#lastActionSuccess = self.event.metadata['lastActionSuccess']
lastActionSuccess = self.event.return_status
print("last action sucess", lastActionSuccess)
self.process_frame()
#print (self.get_pose())
#self.pose = game_util.get_pose(self.event)
print("current pose = ", self.pose)
self.board = None
#point_dists = np.sum(np.abs(self.graph.points - np.array(self.end_point[:2])), axis=1)
#print ("jsut b4 end of function")
#dist_min = np.min(point_dists)
#self.is_possible_end_point = int(dist_min < 0.0001)
print("end of reset in game state function")
def reset(self, scene_num=None, use_gt=True, seed=None, max_num_repeats=constants.MAX_NUM_OF_OBJ_INSTANCES,
remove_prob=None, scene=None, objs=None):
# Reset should be called only when all information from a scene should be cleared.
self.current_frame_count = 0
self.timers = np.zeros((2, 2))
self.board = None
self.original_board = None
self.currently_opened_object_ids = SetWithGet()
self.cleaned_object_ids = SetWithGet()
self.hot_object_ids = SetWithGet()
self.cool_object_ids = SetWithGet()
self.on_object_ids = set()
self.toggleable_object_ids = set()
self.sliced_object_ids = set()
self.inventory_ids = SetWithGet()
self.scene_name = None
self.bounds = None
self.start_point = None
self.event = None
self.s_t = None
self.s_t_orig = None
self.s_t_depth = None
self.pose = None
self.agent_height = None
self.camera_height = None
new_scene = (self.gt_graph is None or self.gt_graph.scene_id is None or self.gt_graph.scene_id == scene_num)
self.scene_num = scene_num
if self.scene_num is None:
self.scene_num = self.local_random.choice(constants.SCENE_NUMBERS)
self.scene_num = self.scene_num
if scene is not None:
self.scene_num = scene['scene_num']
seed = scene['random_seed'] % 1000000
self.scene_name = 'FloorPlan%d' % self.scene_num
self.event = self.env.reset(self.scene_name)
if max_num_repeats is None:
self.event = self.env.random_initialize(seed)
else:
self.env.step(dict(
action='Initialize',
gridSize=constants.AGENT_STEP_SIZE / constants.RECORD_SMOOTHING_FACTOR,
cameraY=constants.CAMERA_HEIGHT_OFFSET,
renderImage=constants.RENDER_IMAGE,
renderDepthImage=constants.RENDER_DEPTH_IMAGE,
renderClassImage=constants.RENDER_CLASS_IMAGE,
renderObjectImage=constants.RENDER_OBJECT_IMAGE,
visibility_distance=constants.VISIBILITY_DISTANCE,
makeAgentsVisible=False,
))
free_per_receptacle = []
if objs is not None:
if 'sparse' in objs:
for o, c in objs['sparse']:
free_per_receptacle.append({'objectType': o, 'count': c})
if 'empty' in objs:
for o, c in objs['empty']:
free_per_receptacle.append({'objectType': o, 'count': c})
self.env.step(dict(action='InitialRandomSpawn', randomSeed=seed, forceVisible=True,
numDuplicatesOfType=[{'objectType': o, 'count': c}
for o, c in objs['repeat']]
if objs is not None and 'repeat' in objs else None,
excludedReceptacles=[obj['objectType'] for obj in free_per_receptacle]
))
# if 'clean' action, make everything dirty and empty out fillable things
if constants.pddl_goal_type == "pick_clean_then_place_in_recep":
# need to create a dictionary that contains all the object's type and state change.
for o in objs['repeat']:
self.env.step(dict(action='SetObjectStates',
SetObjectStates={'objectType': o[0], 'stateChange': 'dirtyable', 'isDirty': True}))
self.env.step(dict(action='SetObjectStates',
SetObjectStates={'objectType': o[0], 'stateChange': 'canFillWithLiquid', 'isFilledWithLiquid': False}))
if objs is not None and ('seton' in objs and len(objs['seton']) > 0):
for o, v in objs['seton']:
self.env.step(dict(action='SetObjectStates', SetObjectStates={'objectType': o, 'stateChange': 'toggleable', 'isToggled': v}))
self.gt_graph = graph_obj.Graph(use_gt=True, construct_graph=True, scene_id=self.scene_num)
if seed is not None:
self.local_random.seed(seed)
print('set seed in game_state_base reset', seed)
self.bounds = np.array([self.gt_graph.xMin, self.gt_graph.yMin,
self.gt_graph.xMax - self.gt_graph.xMin + 1,
self.gt_graph.yMax - self.gt_graph.yMin + 1])
self.agent_height = self.env.last_event.metadata['agent']['position']['y']
self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET
start_point = self.local_random.randint(0, self.gt_graph.points.shape[0] - 1)
start_point = self.gt_graph.points[start_point, :].copy()
self.start_point = (start_point[0], start_point[1], self.local_random.randint(0, 3))
action = {'action': 'TeleportFull',
'x': self.start_point[0] * constants.AGENT_STEP_SIZE,
'y': self.agent_height,
'z': self.start_point[1] * constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'horizon': 30,
'rotation': self.start_point[2] * 90,
}
self.event = self.env.step(action)
constants.data_dict['scene']['scene_num'] = self.scene_num
constants.data_dict['scene']['init_action'] = action
constants.data_dict['scene']['floor_plan'] = self.scene_name
constants.data_dict['scene']['random_seed'] = seed
self.pose = game_util.get_pose(self.event)
# Manually populate parentReceptacles data based on existing ReceptableObjectIds.
objects = self.env.last_event.metadata['objects']
for idx in range(len(objects)):
if objects[idx]['parentReceptacles'] is None:
objects[idx]['parentReceptacles'] = []
for jdx in range(len(objects)):
if (objects[jdx]['receptacleObjectIds'] is not None
and objects[idx]['objectId'] in objects[jdx]['receptacleObjectIds']):
objects[idx]['parentReceptacles'].append(objects[jdx]['objectId'])
def reset(self, scene_name=None, use_gt=True, seed=None):
if scene_name is None:
# Do half reset
action_ind = self.local_random.randint(
0, constants.STEPS_AHEAD**2 - 1)
action_x = action_ind % constants.STEPS_AHEAD - int(
constants.STEPS_AHEAD / 2)
action_z = int(action_ind / constants.STEPS_AHEAD) + 1
x_shift = 0
z_shift = 0
if self.pose[2] == 0:
x_shift = action_x
z_shift = action_z
elif self.pose[2] == 1:
x_shift = action_z
z_shift = -action_x
elif self.pose[2] == 2:
x_shift = -action_x
z_shift = -action_z
elif self.pose[2] == 3:
x_shift = -action_z
z_shift = action_x
action_x = self.pose[0] + x_shift
action_z = self.pose[1] + z_shift
self.end_point = (action_x, action_z, self.pose[2])
else:
# Do full reset
self.scene_name = scene_name
grid_file = 'layouts/%s-layout.npy' % scene_name
self.graph = graph_obj.Graph(grid_file, use_gt=use_gt)
if seed is not None:
self.local_random.seed(seed)
lastActionSuccess = False
self.bounds = [
self.graph.xMin, self.graph.yMin,
self.graph.xMax - self.graph.xMin + 1,
self.graph.yMax - self.graph.yMin + 1
]
while not lastActionSuccess:
self.event = game_util.reset(self.env, self.scene_name)
self.agent_height = self.event.metadata['agent']['position'][
'y']
self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET
self.event = self.env.random_initialize(seed)
start_point = self.local_random.randint(
0, self.graph.points.shape[0] - 1)
start_point = self.graph.points[start_point, :].copy()
self.start_point = (start_point[0], start_point[1],
self.local_random.randint(0, 3))
self.end_point = self.start_point
while self.end_point[0] == self.start_point[
0] and self.end_point[1] == self.start_point[1]:
end_point = self.local_random.randint(
0, self.graph.points.shape[0] - 1)
end_point = self.graph.points[end_point, :].copy()
self.end_point = [
end_point[0], end_point[1],
self.local_random.randint(0, 3)
]
self.end_point[0] += self.local_random.randint(
-constants.TERMINAL_CHECK_PADDING,
constants.TERMINAL_CHECK_PADDING)
self.end_point[1] += self.local_random.randint(
-constants.TERMINAL_CHECK_PADDING,
constants.TERMINAL_CHECK_PADDING)
self.end_point = tuple(self.end_point)
action = {
'action': 'TeleportFull',
'x': self.start_point[0] * constants.AGENT_STEP_SIZE,
'y': self.agent_height,
'z': self.start_point[1] * constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'rotation': self.start_point[2] * 90,
'horizon': 60
}
self.event = self.env.step(action)
lastActionSuccess = self.event.metadata['lastActionSuccess']
self.process_frame()
self.board = None
point_dists = np.sum(np.abs(self.graph.points -
np.array(self.end_point[:2])),
axis=1)
dist_min = np.min(point_dists)
self.is_possible_end_point = int(dist_min < 0.0001)
def create_dump():
time_str = py_util.get_time_str()
prefix = 'questions/'
if not os.path.exists(prefix + dataset_type + '/data_counting'):
os.makedirs(prefix + dataset_type + '/data_counting')
h5 = h5py.File(
prefix + dataset_type + '/data_counting/Counting_Questions_' +
time_str + '.h5', 'w')
h5.create_dataset('questions/question', (num_record, 4),
dtype=np.int32)
print('Generating %d counting questions' % num_record)
# Generate counting questions
data_ind = 0
episode = Episode()
scene_number = random.randint(0, len(scene_numbers) - 1)
while data_ind < num_record:
k = 0
scene_number += 1
scene_num = scene_numbers[scene_number % len(scene_numbers)]
scene_name = 'FloorPlan%d' % scene_num
episode.initialize_scene(scene_name)
num_tries = 0
while num_tries < num_samples_per_scene:
# randomly pick a pickable object in the scene
object_class = random.choice(all_object_classes)
question = CountQuestion(
object_class
) # randomly generate a general counting question
generated = [None] * (constants.MAX_COUNTING_ANSWER + 1)
generated_counts = set()
num_tries += 1
grid_file = 'layouts/%s-layout.npy' % scene_name
xray_graph = graph_obj.Graph(grid_file,
use_gt=True,
construct_graph=False)
scene_bounds = [
xray_graph.xMin, xray_graph.yMin,
xray_graph.xMax - xray_graph.xMin + 1,
xray_graph.yMax - xray_graph.yMin + 1
]
for i in range(100):
if DEBUG:
print('starting try ', i)
scene_seed = random.randint(0, 999999999)
episode.initialize_episode(
scene_seed=scene_seed,
max_num_repeats=constants.MAX_COUNTING_ANSWER + 1,
remove_prob=0.5)
answer = question.get_answer(episode)
object_target = constants.OBJECT_CLASS_TO_ID[object_class]
if answer > 0 and answer not in generated_counts:
if DEBUG:
print('target', str(question), object_target,
answer)
event = episode.event
# Make sure findable
try:
objs = {
obj['objectId']: obj
for obj in event.metadata['objects']
if obj['objectType'] == object_class
}
xray_graph.memory[:, :, 1:] = 0
for obj in objs.values():
obj_point = game_util.get_object_point(
obj, scene_bounds)
xray_graph.memory[obj_point[1], obj_point[0],
object_target + 1] = 1
start_graph = xray_graph.memory.copy()
graph_points = xray_graph.points.copy()
graph_points = graph_points[np.random.permutation(
graph_points.shape[0]), :]
num_checked_points = 0
point_ind = 0
# Initial check to make sure all objects are visible on the grid.
while point_ind < len(graph_points):
start_point = graph_points[point_ind]
headings = np.random.permutation(4)
for heading in headings:
start_point = (start_point[0],
start_point[1], heading)
patch = xray_graph.get_graph_patch(
start_point)[0][:, :,
object_target + 1]
if patch.max() > 0:
point_ind = 0
xray_graph.update_graph(
(np.zeros((constants.STEPS_AHEAD,
constants.STEPS_AHEAD,
1)), 0), start_point,
[object_target + 1])
point_ind += 1
if np.max(xray_graph.memory[:, :, object_target +
1]) > 0:
if DEBUG:
print('some points could not be reached')
answer = None
raise AssertionError
xray_graph.memory = start_graph
point_ind = 0
seen_objs = set()
while point_ind < len(graph_points):
start_point = graph_points[point_ind]
headings = np.random.permutation(4)
for heading in headings:
start_point = (start_point[0],
start_point[1], heading)
patch = xray_graph.get_graph_patch(
start_point)[0]
if patch[:, :,
object_target + 1].max() > 0:
action = {
'action': 'TeleportFull',
'x': start_point[0] *
constants.AGENT_STEP_SIZE,
'y': episode.agent_height,
'z': start_point[1] *
constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'rotation': start_point[2] * 90,
'horizon': -30,
}
event = episode.env.step(action)
num_checked_points += 1
if num_checked_points > 20:
if DEBUG:
print('timeout')
answer = None
raise AssertionError
changed = False
for jj in range(4):
open_success = True
opened_objects = set()
parents = [
game_util.get_object(
obj['parentReceptacle'],
event.metadata)
for obj in objs.values()
]
openable_parents = [
parent for parent in parents
if parent['visible']
and parent['openable']
and not parent['isopen']
]
while open_success:
obj_list = list(objs.values())
for obj in obj_list:
if obj['objectId'] in event.instance_detections2D:
if game_util.check_object_size(
event.
instance_detections2D[
obj['objectId']]
):
seen_objs.add(
obj['objectId']
)
if DEBUG:
print(
'seen',
seen_objs)
del objs[obj[
'objectId']]
changed = True
num_checked_points = 0
if len(seen_objs
) == answer:
raise AssertionError
if len(openable_parents) > 0:
action = {
'action': 'OpenObject'
}
game_util.set_open_close_object(
action, event)
event = episode.env.step(
action)
open_success = event.metadata[
'lastActionSuccess']
if open_success:
opened_objects.add(
episode.env.
last_action[
'objectId'])
else:
open_success = False
for opened in opened_objects:
event = episode.env.step({
'action':
'CloseObject',
'objectId':
opened,
'forceVisible':
True
})
if not event.metadata[
'lastActionSuccess']:
answer = None
raise AssertionError
if jj < 3:
event = episode.env.step(
{'action': 'LookDown'})
if changed:
point_ind = 0
num_checked_points = 0
xray_graph.memory[:, :,
object_target +
1] = 0
for obj in objs.values():
obj_point = game_util.get_object_point(
obj, scene_bounds)
xray_graph.memory[
obj_point[1], obj_point[0],
object_target + 1] = 1
point_ind += 1
if DEBUG:
print('ran out of points')
answer = None
except AssertionError:
if answer is not None:
if DEBUG:
print('success')
pass
print(str(question), object_target, answer)
if answer is not None and answer < len(
generated) and answer not in generated_counts:
generated[answer] = [
scene_num, scene_seed,
constants.OBJECT_CLASS_TO_ID[object_class], answer
]
generated_counts.add(answer)
print('\tcounts', sorted(list(generated_counts)))
if len(generated_counts) == len(generated):
for q in generated:
if data_ind >= h5['questions/question'].shape[0]:
num_tries = 2**32
break
h5['questions/question'][data_ind, :] = np.array(q)
data_ind += 1
k += 1
h5.flush()
break
print("# generated samples: {}".format(data_ind))
h5.close()
episode.env.stop_unity()
def augment_traj(env, json_file):
# load json data
with open(json_file) as f:
traj_data = json.load(f)
# make directories
root_dir = json_file.replace(TRAJ_DATA_JSON_FILENAME, "")
# fresh images list
traj_data['images'] = list()
# scene setup
scene_num = traj_data['scene']['scene_num']
object_poses = traj_data['scene']['object_poses']
object_toggles = traj_data['scene']['object_toggles']
dirty_and_empty = traj_data['scene']['dirty_and_empty']
# reset
scene_name = 'FloorPlan%d' % scene_num
env.reset(scene_name)
env.restore_scene(object_poses, object_toggles, dirty_and_empty)
print(scene_name)
env.step(dict(traj_data['scene']['init_action']))
print("Task: %s" % (traj_data['turk_annotations']['anns'][0]['task_desc']))
# setup task
env.set_task(traj_data, args, reward_type='dense')
game_state = TextWorldTaskGameStateFullKnowledge(env)
# reset
game_state.receptacle_to_point = None
game_state.task_target = None
game_state.success = False
# load nav graph
game_state.gt_graph = graph_obj.Graph(use_gt=True,
construct_graph=True,
scene_id=scene_num)
game_state.gt_graph.clear()
game_state.agent_height = env.last_event.metadata['agent']['position']['y']
game_state.camera_height = game_state.agent_height + constants.CAMERA_HEIGHT_OFFSET
points_source = 'gen/layouts/%s-openable.json' % scene_name
with open(points_source, 'r') as f:
openable_object_to_point = json.load(f)
game_state.openable_object_to_point = openable_object_to_point
pddl_params = traj_data['pddl_params']
game_state.object_target = constants.OBJECTS.index(
pddl_params['object_target']) if pddl_params['object_target'] else None
game_state.parent_target = constants.OBJECTS.index(
pddl_params['parent_target']) if pddl_params['parent_target'] else None
game_state.mrecep_target = constants.OBJECTS.index(
pddl_params['mrecep_target']) if pddl_params['mrecep_target'] else None
game_state.toggle_target = constants.OBJECTS.index(
pddl_params['toggle_target']) if pddl_params['toggle_target'] else None
game_state.task_target = (game_state.object_target,
game_state.parent_target,
game_state.toggle_target,
game_state.mrecep_target)
game_state.update_receptacle_nearest_points()
pddl_str = game_state.state_to_pddl(traj_data)
pddl_file = os.path.join(os.path.dirname(json_file), 'initial_state.pddl')
with open(pddl_file, 'w') as f:
f.write(pddl_str)
print("Wrote to %s" % pddl_file)
game_state.planner.process_pool.terminate()
def reset(self, seed=None, test_ind=None):
if self.game_state.env is not None:
self.game_state.reset(seed=seed, test_ind=test_ind)
self.bounds = self.game_state.bounds
self.end_points = np.zeros_like(self.game_state.graph.memory[:, :, 0])
for end_point in self.game_state.end_point:
self.end_points[end_point[1] - self.game_state.graph.yMin,
end_point[0] - self.game_state.graph.xMin] = 1
self.gru_state = np.zeros((1, constants.RL_GRU_SIZE))
self.pose = self.game_state.pose
self.prev_pose = self.pose
self.visited_poses = set()
self.reward = 0
self.num_steps = 0
self.question_count = 0
self.num_invalid_actions = 0
self.prev_can_end = False
dilation_kernel = np.ones(
(constants.SCENE_PADDING * 2 + 1, constants.SCENE_PADDING))
free_space = self.game_state.xray_graph.memory.copy().squeeze()
if len(free_space.shape) == 3:
free_space = free_space[:, :, 0]
free_space[free_space == graph_obj.MAX_WEIGHT] = 0
free_space[free_space > 1] = 1
self.dilation = np.zeros_like(free_space)
for _ in range(2):
self.dilation += scipy.ndimage.morphology.binary_dilation(
free_space, structure=dilation_kernel).astype(np.int)
dilation_kernel = np.rot90(dilation_kernel)
self.dilation[self.dilation > 1] = 1
self.max_coverage = np.sum(self.dilation)
# Rows are:
# 0 - Map weights (not fed to decision network)
# 1 and 2 - meshgrid
# 3 - coverage
# 4 - teleport locations
# 5 - free space map
# 6 - visited locations
# 7+ - object location
if constants.USE_NAVIGATION_AGENT:
if self.nav_agent is not None:
self.nav_agent.reset(self.game_state.scene_name)
self.spatial_map = graph_obj.Graph('layouts/%s-layout.npy' %
self.game_state.scene_name,
use_gt=True,
construct_graph=False)
self.spatial_map.memory = np.concatenate(
(np.zeros((self.bounds[3], self.bounds[2], 7)),
self.game_state.graph.memory[:, :, 1:].copy()),
axis=2)
self.coverage = 0
self.terminal = False
self.new_coverage = 0
self.last_meta_action = np.zeros(7)
self.last_action_one_hot = np.zeros(
self.network.pi.get_shape().as_list()[-1])
self.global_step_id += 1
self.update_spatial_map({'action': 'Initialize'})
# For drawing
self.forward_pred = np.zeros((3, 3, 3))
self.next_memory_crops_rot = np.zeros((3, 3, 3))
def create_dump():
episode = Episode()
count = 0
#for i in range(len(scene_numbers)):
while count > -1:
scene_name = 'FloorPlan%d' % scene_numbers[count]
print('scene is %s' % scene_name)
grid_file = 'layouts/%s-layout.npy' % scene_name
graph = graph_obj.Graph(grid_file) #, use_gt=use_gt)
print("graph: ", graph)
scene_bounds = [
graph.xMin, graph.yMin, graph.xMax - graph.xMin + 1,
graph.yMax - graph.yMin + 1
]
print("bounds: ", scene_bounds)
episode.initialize_scene(scene_name)
#scene_seed = random.randint(0, 999999999)
#episode.initialize_episode(scene_seed=scene_seed) # randomly initialize the scene
prefix = 'object_semantic_map/maps/'
for index in range(1, 11):
#memory = np.zeros((graph.yMax - graph.yMin + 1, graph.xMax - graph.xMin + 1, 1 + constants.NUM_CLASSES))
npy_path = os.path.join(prefix, str(index) + '.npy')
memory = np.load(npy_path)
print(memory.shape)
"""
for _ in range(SIM_TIMES):
scene_seed = random.randint(0, 999999999)
episode.initialize_episode(scene_seed=scene_seed)
for obj in episode.get_objects():
if obj['objectType'] not in constants.OBJECTS:
continue
y, x = game_util.get_object_point(obj, scene_bounds)
#print("object %s at " % obj['objectType'], game_util.get_object_point(obj, scene_bounds))
obj_id = constants.OBJECT_CLASS_TO_ID[obj['objectType']]
memory[y][x][obj_id] += 1
memory = np.divide(memory, SIM_TIMES)
"""
print(memory.shape[:2])
#print (constants.OBJECTS[test_id])
#plt.figure(figsize=list(memory.shape[:2]))
#plt.pcolor(memory[:,:,test_id],cmap=plt.get_cmap('Reds'), vmin=0.0, vmax=1.0)#,edgecolors='k', linewidths=1)
#plt.colorbar()
#print (memory[:,:,test_id])
#plt.show()
fig, axs = plt.subplots(4, 7)
#a1
for (i, j) in list(itertools.product("0123", "23456")):
#print (i, j)
obj_id = (int(i)) * 5 + int(j) - 1
#if obj_id > 20: break
ax = axs[int(i), int(j)]
pcm = ax.pcolor(memory[:, :, obj_id],
cmap=plt.get_cmap('Reds'),
vmin=-0,
vmax=1)
ax.set_title(constants.OBJECTS[obj_id])
ax.axis('off')
gs = axs[0, 0].get_gridspec()
for ax in axs[:, :2].flatten():
ax.remove()
ax_frame = fig.add_subplot(gs[:, :2])
ax_frame.axis('off')
new_frame = get_agent_map_data(episode.env)
new_frame = new_frame[:, ::-1]
new_frame = np.rot90(new_frame, -1)
plt.imshow(new_frame)
#fig.tight_layout()
#fig.colorbar(pcm, ax=axs[:])
fig_path = os.path.join(prefix, str(index) + '.png')
plt.savefig(fig_path)
"""
objs = [obj['objectType'] for obj in episode.get_objects()]
objjs = [obj['objectId'] for obj in episode.get_objects()]
objjspos = [obj['position'] for obj in episode.get_objects()]
for ii in range(len(episode.get_objects())):
print("object%d at " % ii, game_util.get_object_point(episode.get_objects()[ii], scene_bounds))
print("objs: ", objs)
print("objjs: ", objjs)
print("objjs: ", len(objjs))
print("objjspos: ", objjspos)
print("objjspos: ", len(objjspos))
"""
#event = episode.get_env_top_view()
#print (event.metadata)
#image = drawing.subplot()
break
