def load_nav_graph(self): ''' build navigation grid graph ''' floor_plan = self.traj['scene']['floor_plan'] scene_num = self.traj['scene']['scene_num'] self.gt_graph = graph_obj.Graph(use_gt=True, construct_graph=True, scene_id=scene_num)
def reset(self, scene_name=None, seed=None): if scene_name is not None: if self.game_state.env is not None and type( self.game_state) == GameState: self.game_state.reset(scene_name, use_gt=False, seed=seed) self.gt_graph = graph_obj.Graph('layouts/%s-layout.npy' % scene_name, use_gt=True) self.bounds = [ self.game_state.graph.xMin, self.game_state.graph.yMin, self.game_state.graph.xMax - self.game_state.graph.xMin + 1, self.game_state.graph.yMax - self.game_state.graph.yMin + 1 ] if len(self.game_state.end_point) == 0: self.game_state.end_point = (self.game_state.graph.xMin + constants.TERMINAL_CHECK_PADDING, self.game_state.graph.yMin + constants.TERMINAL_CHECK_PADDING, 0) self.action = np.zeros(self.action_util.num_actions) self.memory = np.zeros( (constants.SPATIAL_MAP_HEIGHT, constants.SPATIAL_MAP_WIDTH, constants.MEMORY_SIZE)) self.gru_state = np.zeros((1, constants.GRU_SIZE)) self.pose = self.game_state.pose self.is_possible = 1 self.num_steps = 0 self.times = np.zeros(2) self.impossible_spots = set() self.visited_spots = set() else: self.game_state.reset() self.goal_pose = np.array([ self.game_state.end_point[0] - self.game_state.graph.xMin, self.game_state.end_point[1] - self.game_state.graph.yMin ], dtype=np.int32)[:2] self.inference()
def reset(self, scene_name=None, seed=None, config_filename=""): if scene_name is not None: if self.game_state.env is not None and type( self.game_state) == GameState: self.game_state.reset(scene_name, use_gt=False, seed=seed, config_filename=config_filename) self.gt_graph = graph_obj.Graph( 'layouts/%s-layout_%s.npy' % (scene_name, str(constants.AGENT_STEP_SIZE)), self.action_util, use_gt=True) self.bounds = [ self.game_state.graph.xMin, self.game_state.graph.yMin, self.game_state.graph.xMax - self.game_state.graph.xMin + 1, self.game_state.graph.yMax - self.game_state.graph.yMin + 1 ] ''' if len(self.game_state.end_point) == 0: self.game_state.end_point = (self.game_state.graph.xMin + constants.TERMINAL_CHECK_PADDING, self.game_state.graph.yMin + constants.TERMINAL_CHECK_PADDING, 0) print ("in agent reset end point set to : " , self.game_state.end_point) ''' self.action = np.zeros(self.action_util.num_actions) self.memory = np.zeros( (constants.SPATIAL_MAP_HEIGHT, constants.SPATIAL_MAP_WIDTH, constants.MEMORY_SIZE)) self.gru_state = np.zeros((1, constants.GRU_SIZE)) self.pose = self.game_state.pose self.is_possible = 1 self.num_steps = 0 self.times = np.zeros(2) self.impossible_spots = set() self.visited_spots = set() else: #print ("") self.game_state.reset()
def create_dump(): time_str = py_util.get_time_str() prefix = 'questions/' if not os.path.exists(prefix + dataset_type + '/data_contains'): os.makedirs(prefix + dataset_type + '/data_contains') h5 = h5py.File(prefix + dataset_type + '/data_contains/Contains_Questions_' + time_str + '.h5', 'w') h5.create_dataset('questions/question', (num_record, 5), dtype=np.int32) print('Generating %d contains questions' % num_record) # Generate contains questions data_ind = 0 episode = Episode() scene_number = -1 while data_ind < num_record: k = 0 scene_number += 1 scene_num = scene_numbers[scene_number % len(scene_numbers)] scene_name = 'FloorPlan%d' % scene_num episode.initialize_scene(scene_name) num_tries = 0 while k < num_samples_per_scene and num_tries < 10 * num_samples_per_scene: # randomly pick a pickable object in the scene object_class = random.choice(all_object_classes) generated_no, generated_yes = False, False question = None temp_data = [] num_tries += 1 grid_file = 'layouts/%s-layout.npy' % scene_name xray_graph = graph_obj.Graph(grid_file, use_gt=True, construct_graph=False) scene_bounds = [xray_graph.xMin, xray_graph.yMin, xray_graph.xMax - xray_graph.xMin + 1, xray_graph.yMax - xray_graph.yMin + 1] for i in range(20): # try 20 times scene_seed = random.randint(0, 999999999) episode.initialize_episode(scene_seed=scene_seed) # randomly initialize the scene if not question: question = ExistenceQuestion.get_true_contain_question(episode, all_object_classes, receptacles) if DEBUG: print(str(question)) if not question: continue object_class = question.object_class parent_class = question.parent_object_class answer = question.get_answer(episode) object_target = constants.OBJECT_CLASS_TO_ID[object_class] if answer and not generated_yes: event = episode.event xray_graph.memory[:, :, 1:] = 0 objs = {obj['objectId']: obj for obj in event.metadata['objects'] if obj['objectType'] == object_class and obj['parentReceptacle'].split('|')[0] == parent_class} for obj in objs.values(): if obj['objectType'] != object_class: continue obj_point = game_util.get_object_point(obj, scene_bounds) xray_graph.memory[obj_point[1], obj_point[0], constants.OBJECT_CLASS_TO_ID[obj['objectType']] + 1] = 1 # Make sure findable try: graph_points = xray_graph.points.copy() graph_points = graph_points[np.random.permutation(graph_points.shape[0]), :] num_checked_points = 0 for start_point in graph_points: headings = np.random.permutation(4) for heading in headings: start_point = (start_point[0], start_point[1], heading) patch = xray_graph.get_graph_patch(start_point)[0] if patch[:, :, object_target + 1].max() > 0: action = {'action': 'TeleportFull', 'x': start_point[0] * constants.AGENT_STEP_SIZE, 'y': episode.agent_height, 'z': start_point[1] * constants.AGENT_STEP_SIZE, 'rotateOnTeleport': True, 'rotation': start_point[2] * 90, 'horizon': -30, } event = episode.env.step(action) num_checked_points += 1 if num_checked_points > 1000: answer = None raise AssertionError for jj in range(4): open_success = True opened_objects = set() parents = [game_util.get_object(obj['parentReceptacle'], event.metadata) for obj in objs.values()] openable_parents = [parent for parent in parents if parent['visible'] and parent['openable'] and not parent['isopen']] while open_success: for obj in objs.values(): if obj['objectId'] in event.instance_detections2D: if game_util.check_object_size(event.instance_detections2D[obj['objectId']]): raise AssertionError if len(openable_parents) > 0: action = {'action': 'OpenObject'} game_util.set_open_close_object(action, event) event = episode.env.step(action) open_success = event.metadata['lastActionSuccess'] if open_success: opened_objects.add(episode.env.last_action['objectId']) else: open_success = False for opened in opened_objects: event = episode.env.step({ 'action': 'CloseObject', 'objectId': opened, 'forceVisible': True}) if not event.metadata['lastActionSuccess']: answer = None raise AssertionError if jj < 3: event = episode.env.step({'action': 'LookDown'}) answer = None except AssertionError: if answer is None and DEBUG: print('failed to generate') print(str(question), answer) if answer == False and not generated_no: generated_no = True temp_data.append([scene_num, scene_seed, constants.OBJECT_CLASS_TO_ID[object_class], constants.OBJECT_CLASS_TO_ID[parent_class], answer]) elif answer == True and not generated_yes: generated_yes = True temp_data.append([scene_num, scene_seed, constants.OBJECT_CLASS_TO_ID[object_class], constants.OBJECT_CLASS_TO_ID[parent_class], answer]) if generated_no and generated_yes: h5['questions/question'][data_ind, :] = np.array(temp_data[0]) h5['questions/question'][data_ind + 1, :] = np.array(temp_data[1]) h5.flush() data_ind += 2 k += 2 break print("# generated samples: {}".format(data_ind)) h5.close() episode.env.stop_unity()
def reset(self, seed=None, test_ind=None): self.board = None self.seen_object = False self.terminal = False self.opened_receptacles = set() self.closed_receptacles = set() self.seen_obj1 = set() self.seen_obj2 = set() self.visited_locations = set() self.can_end = False if seed is not None: self.local_random.seed(seed) # Do equal number of each question type in train. question_type_ind = self.local_random.sample( constants.USED_QUESTION_TYPES, 1)[0] # get random row if test_ind is not None: question_row, question_type_ind = test_ind question_type = self.question_types[question_type_ind] question_data = self.test_datasets[question_type_ind][ question_row, :] test_ind = (question_row, question_type_ind) else: question_type = self.question_types[question_type_ind] question_row = self.local_random.randint( 0, len(self.datasets[question_type_ind]) - 1) question_data = self.datasets[question_type_ind][question_row, :] container_ind = None if question_type_ind == 0 or question_type_ind == 1: scene_num, scene_seed, object_ind, answer = question_data self.question_target = object_ind if question_type_ind == 0: answer = bool(answer) elif question_type_ind == 2: scene_num, scene_seed, object_ind, container_ind, answer = question_data answer = bool(answer) self.question_target = (object_ind, container_ind) else: raise Exception('No question type found for type %d' % question_type_ind) self.scene_seed = scene_seed self.scene_num = scene_num self.object_target = object_ind self.parent_target = container_ind self.container_target = np.zeros(constants.NUM_CLASSES) self.direction_target = np.zeros(4) if container_ind is not None: self.container_target[container_ind] = 1 self.question_type_ind = question_type_ind self.scene_name = 'FloorPlan%d' % scene_num grid_file = 'layouts/%s-layout.npy' % self.scene_name self.graph = graph_obj.Graph(grid_file, use_gt=False) self.xray_graph = graph_obj.Graph(grid_file, use_gt=True) self.bounds = [ self.graph.xMin, self.graph.yMin, self.graph.xMax - self.graph.xMin + 1, self.graph.yMax - self.graph.yMin + 1 ] max_num_repeats = 1 remove_prob = 0.5 if question_type == 'existence': max_num_repeats = 10 remove_prob = 0.25 elif question_type == 'counting': max_num_repeats = constants.MAX_COUNTING_ANSWER + 1 remove_prob = 0.5 elif question_type == 'contains': max_num_repeats = 10 remove_prob = 0.25 self.event = game_util.reset(self.env, self.scene_name) self.agent_height = self.event.metadata['agent']['position']['y'] self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET self.event = self.env.random_initialize( self.scene_seed, max_num_repeats=max_num_repeats, remove_prob=remove_prob) print('Type:', question_type, 'Row: ', question_row, 'Scene', self.scene_name, 'seed', scene_seed) print( 'Question:', game_util.get_question_str(question_type_ind, object_ind, container_ind)) if self.question_type_ind == 2: print('Answer:', constants.OBJECTS[object_ind], 'in', constants.OBJECTS[container_ind], 'is', answer) else: print('Answer:', constants.OBJECTS[object_ind], 'is', answer) self.answer = answer # Verify answer if self.question_type_ind == 0: objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind], self.event.metadata) computed_answer = len(objs) > 0 requires_interaction = True for obj in objs: parent = obj['parentReceptacle'].split('|')[0] if parent not in {'Fridge', 'Cabinet', 'Microwave'}: requires_interaction = False break elif self.question_type_ind == 1: objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind], self.event.metadata) computed_answer = len(objs) requires_interaction = True elif self.question_type_ind == 2: objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind], self.event.metadata) if len(objs) == 0: computed_answer = False requires_interaction = constants.OBJECTS[ self.question_target[1]] in { 'Fridge', 'Cabinet', 'Microwave' } else: obj = objs[0] computed_answer = False for obj in objs: requires_interaction = True parent = obj['parentReceptacle'].split('|')[0] if parent in constants.OBJECT_CLASS_TO_ID: parent_ind = constants.OBJECT_CLASS_TO_ID[parent] computed_answer = parent_ind == self.question_target[1] if computed_answer: if parent not in { 'Fridge', 'Cabinet', 'Microwave' }: requires_interaction = False break else: computed_answer = False self.requires_interaction = requires_interaction try: assert self.answer == computed_answer, 'Answer does not match scene metadata' except AssertionError: print('Type:', question_type, 'Row: ', question_row, 'Scene', self.scene_name, 'seed', scene_seed) print('Answer', computed_answer, 'does not match expected value', self.answer, ', did randomization process change?') pdb.set_trace() self.answer = computed_answer if constants.NUM_CLASSES > 1: self.hidden_items = set() objects = self.event.metadata['objects'] for obj in objects: if obj['receptacle'] and obj['openable'] and not obj['isopen']: for inside_obj in obj['receptacleObjectIds']: self.hidden_items.add(inside_obj) objects = self.event.metadata['objects'] for obj in objects: if obj['objectType'] not in constants.OBJECT_CLASS_TO_ID: continue obj_bounds = game_util.get_object_bounds(obj, self.bounds) self.xray_graph.memory[ obj_bounds[1]:obj_bounds[3], obj_bounds[0]:obj_bounds[2], constants.OBJECT_CLASS_TO_ID[obj['objectType']] + 1] = 1 start_point = self.local_random.randint(0, self.graph.points.shape[0] - 1) start_point = self.graph.points[start_point, :].copy() self.start_point = (start_point[0], start_point[1], self.local_random.randint(0, 3)) action = { 'action': 'TeleportFull', 'x': self.start_point[0] * constants.AGENT_STEP_SIZE, 'y': self.agent_height, 'z': self.start_point[1] * constants.AGENT_STEP_SIZE, 'rotateOnTeleport': True, 'rotation': self.start_point[2] * 90, 'horizon': 30, } self.event = self.env.step(action) self.process_frame() self.reward = 0 self.end_point = []
def reset(self, scene_name=None, use_gt=True, seed=None, config_filename=""): if scene_name is None: # Do half reset action_ind = self.local_random.randint( 0, constants.STEPS_AHEAD**2 - 1) action_x = action_ind % constants.STEPS_AHEAD - int( constants.STEPS_AHEAD / 2) action_z = int(action_ind / constants.STEPS_AHEAD) + 1 x_shift = 0 z_shift = 0 if self.pose[2] == 0: x_shift = action_x z_shift = action_z elif self.pose[2] == 1: x_shift = action_z z_shift = -action_x elif self.pose[2] == 2: x_shift = -action_x z_shift = -action_z elif self.pose[2] == 3: x_shift = -action_z z_shift = action_x action_x = self.pose[0] + x_shift action_z = self.pose[1] + z_shift self.end_point = (action_x, action_z, self.pose[2]) #print ("in the game state reset end point is : ", self.end_point) else: # Do full reset self.scene_name = scene_name #print ("Full reset - in the first time of load") grid_file = 'layouts/%s-layout_%s.npy' % ( scene_name, str(constants.AGENT_STEP_SIZE)) self.graph = graph_obj.Graph(grid_file, self.action_util, use_gt=use_gt) if seed is not None: self.local_random.seed(seed) lastActionSuccess = False self.discovered_explored = {} self.discovered_objects = [] self.bounds = [ self.graph.xMin, self.graph.yMin, self.graph.xMax - self.graph.xMin + 1, self.graph.yMax - self.graph.yMin + 1 ] ''' while not lastActionSuccess: print ("In the while loop") self.event = game_util.reset(self.env, self.scene_name) self.event = self.event.events[0] self.agent_height = self.event.metadata['agent']['position']['y'] self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET #self.event = self.env.random_initialize(seed) #self.start_point = [] start_point_2 = [] #start_point_2.append(int(self.event.metadata['agent']['position']['x']/constants.AGENT_STEP_SIZE) -self.graph.xMin +1) #start_point_2.append(int(self.event.metadata['agent']['position']['z']/constants.AGENT_STEP_SIZE) - self.graph.yMin +1 ) start_point_2.append(int(self.event.metadata['agent']['position']['x']/constants.AGENT_STEP_SIZE) ) start_point_2.append(int(self.event.metadata['agent']['position']['z']/constants.AGENT_STEP_SIZE) ) start_point_2.append(int(self.event.metadata['agent']['rotation']['y']/90)) print(start_point_2) print ("starting horizon", self.event.metadata['agent']['cameraHorizon']) self.start_point = start_point_2[:] #start_point = self.local_random.randint(0, self.graph.points.shape[0] - 1) #start_point = self.graph.points[start_point, :].copy() #self.start_point = (start_point[0], start_point[1], self.local_random.randint(0, 3)) self.end_point = self.start_point #print ("B4 the while loop for assigning end point", self.start_point) while self.end_point[0] == self.start_point[0] and self.end_point[1] == self.start_point[1]: #end_point = self.local_random.randint(0, self.graph.points.shape[0] - 1) #end_point = self.graph.points[end_point, :].copy() #self.end_point = [end_point[0], end_point[1], self.local_random.randint(0, 3)] #self.end_point[0] += self.local_random.randint(-constants.TERMINAL_CHECK_PADDING, constants.TERMINAL_CHECK_PADDING) #self.end_point[1] += self.local_random.randint(-constants.TERMINAL_CHECK_PADDING, constants.TERMINAL_CHECK_PADDING) #self.end_point = tuple(self.end_point) self.end_point = (self.end_point[0]-1 , self.end_point[1]-1 , self.end_point[2]) print ("In the while loop for assigning end point",self.end_point) action = {'action': 'TeleportFull', 'x': self.start_point[0] * constants.AGENT_STEP_SIZE, 'y': self.agent_height, 'z': self.start_point[1] * constants.AGENT_STEP_SIZE, 'rotateOnTeleport': True, 'rotation': self.start_point[2] * 90, 'horizon': 0} #'horizon': 60} ''' while True: self.event = game_util.reset(self.env, self.scene_name, config_filename) #self.event = self.event.events[0] print("type of event 2 : ", type(self.event)) lastActionSuccess = self.event.return_status break start_point_2 = [] ''' self.agent_height = self.event.metadata['agent']['position']['y'] self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET start_point_2.append(int(self.event.metadata['agent']['position']['x']/constants.AGENT_STEP_SIZE) ) start_point_2.append(int(self.event.metadata['agent']['position']['z']/constants.AGENT_STEP_SIZE) ) start_point_2.append(int(self.event.metadata['agent']['rotation']['y']/90)) ''' print("rotation ", self.event.rotation) rotation_angle = -(self.event.rotation % 90) action = "RotateLook, rotation=%d" % int(rotation_angle) self.goal = self.event.goal #action = {'action':"Pass"} #print ("z before turning ", self.event.position['z']/constants.AGENT_STEP_SIZE) #print ("z before turning ", self.event.position['z']) self.event = self.env.step(action) #print ("z after turning ", self.event.position['z']/constants.AGENT_STEP_SIZE) #print ("z after turning ", self.event.position['z']) self.agent_height = self.event.position['y'] self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET start_point_2.append( math.floor(self.event.position['x'] / constants.AGENT_STEP_SIZE)) start_point_2.append( math.floor(self.event.position['z'] / constants.AGENT_STEP_SIZE)) start_point_2.append(int(self.event.rotation / 90)) #print(start_point_2) self.start_point = start_point_2[:] self.end_point = self.start_point[:] self.end_point = (self.end_point[0], self.end_point[1], (self.end_point[2] + 1) % 4) #print ("end point = ", self.end_point) #print ("start point = ", self.start_point) #print ("last action success", self.event['lastActionSuccess']) #print (self.event) #print ("last action success", self.event.lastActionSuccess) #print ("number of objects : ",len(self.event.metadata['objects'])) #action = {"action": "RotateRight", 'rotatio#n':90} #print ("type of event 3 : ", type(self.event)) #action = {"action": "RotateLeft", 'rotation':90} #action = "RotateLook, rotation=-90" #self.event = self.env.step(action) #print ("type of event 4 : ", type(self.event)) #self.event = self.event.events[0] #mcs_output = wrap_output(self.event) #lastActionSuccess = self.event.metadata['lastActionSuccess'] lastActionSuccess = self.event.return_status print("last action sucess", lastActionSuccess) self.process_frame() #print (self.get_pose()) #self.pose = game_util.get_pose(self.event) print("current pose = ", self.pose) self.board = None #point_dists = np.sum(np.abs(self.graph.points - np.array(self.end_point[:2])), axis=1) #print ("jsut b4 end of function") #dist_min = np.min(point_dists) #self.is_possible_end_point = int(dist_min < 0.0001) print("end of reset in game state function")
def reset(self, scene_num=None, use_gt=True, seed=None, max_num_repeats=constants.MAX_NUM_OF_OBJ_INSTANCES, remove_prob=None, scene=None, objs=None): # Reset should be called only when all information from a scene should be cleared. self.current_frame_count = 0 self.timers = np.zeros((2, 2)) self.board = None self.original_board = None self.currently_opened_object_ids = SetWithGet() self.cleaned_object_ids = SetWithGet() self.hot_object_ids = SetWithGet() self.cool_object_ids = SetWithGet() self.on_object_ids = set() self.toggleable_object_ids = set() self.sliced_object_ids = set() self.inventory_ids = SetWithGet() self.scene_name = None self.bounds = None self.start_point = None self.event = None self.s_t = None self.s_t_orig = None self.s_t_depth = None self.pose = None self.agent_height = None self.camera_height = None new_scene = (self.gt_graph is None or self.gt_graph.scene_id is None or self.gt_graph.scene_id == scene_num) self.scene_num = scene_num if self.scene_num is None: self.scene_num = self.local_random.choice(constants.SCENE_NUMBERS) self.scene_num = self.scene_num if scene is not None: self.scene_num = scene['scene_num'] seed = scene['random_seed'] % 1000000 self.scene_name = 'FloorPlan%d' % self.scene_num self.event = self.env.reset(self.scene_name) if max_num_repeats is None: self.event = self.env.random_initialize(seed) else: self.env.step(dict( action='Initialize', gridSize=constants.AGENT_STEP_SIZE / constants.RECORD_SMOOTHING_FACTOR, cameraY=constants.CAMERA_HEIGHT_OFFSET, renderImage=constants.RENDER_IMAGE, renderDepthImage=constants.RENDER_DEPTH_IMAGE, renderClassImage=constants.RENDER_CLASS_IMAGE, renderObjectImage=constants.RENDER_OBJECT_IMAGE, visibility_distance=constants.VISIBILITY_DISTANCE, makeAgentsVisible=False, )) free_per_receptacle = [] if objs is not None: if 'sparse' in objs: for o, c in objs['sparse']: free_per_receptacle.append({'objectType': o, 'count': c}) if 'empty' in objs: for o, c in objs['empty']: free_per_receptacle.append({'objectType': o, 'count': c}) self.env.step(dict(action='InitialRandomSpawn', randomSeed=seed, forceVisible=True, numDuplicatesOfType=[{'objectType': o, 'count': c} for o, c in objs['repeat']] if objs is not None and 'repeat' in objs else None, excludedReceptacles=[obj['objectType'] for obj in free_per_receptacle] )) # if 'clean' action, make everything dirty and empty out fillable things if constants.pddl_goal_type == "pick_clean_then_place_in_recep": # need to create a dictionary that contains all the object's type and state change. for o in objs['repeat']: self.env.step(dict(action='SetObjectStates', SetObjectStates={'objectType': o[0], 'stateChange': 'dirtyable', 'isDirty': True})) self.env.step(dict(action='SetObjectStates', SetObjectStates={'objectType': o[0], 'stateChange': 'canFillWithLiquid', 'isFilledWithLiquid': False})) if objs is not None and ('seton' in objs and len(objs['seton']) > 0): for o, v in objs['seton']: self.env.step(dict(action='SetObjectStates', SetObjectStates={'objectType': o, 'stateChange': 'toggleable', 'isToggled': v})) self.gt_graph = graph_obj.Graph(use_gt=True, construct_graph=True, scene_id=self.scene_num) if seed is not None: self.local_random.seed(seed) print('set seed in game_state_base reset', seed) self.bounds = np.array([self.gt_graph.xMin, self.gt_graph.yMin, self.gt_graph.xMax - self.gt_graph.xMin + 1, self.gt_graph.yMax - self.gt_graph.yMin + 1]) self.agent_height = self.env.last_event.metadata['agent']['position']['y'] self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET start_point = self.local_random.randint(0, self.gt_graph.points.shape[0] - 1) start_point = self.gt_graph.points[start_point, :].copy() self.start_point = (start_point[0], start_point[1], self.local_random.randint(0, 3)) action = {'action': 'TeleportFull', 'x': self.start_point[0] * constants.AGENT_STEP_SIZE, 'y': self.agent_height, 'z': self.start_point[1] * constants.AGENT_STEP_SIZE, 'rotateOnTeleport': True, 'horizon': 30, 'rotation': self.start_point[2] * 90, } self.event = self.env.step(action) constants.data_dict['scene']['scene_num'] = self.scene_num constants.data_dict['scene']['init_action'] = action constants.data_dict['scene']['floor_plan'] = self.scene_name constants.data_dict['scene']['random_seed'] = seed self.pose = game_util.get_pose(self.event) # Manually populate parentReceptacles data based on existing ReceptableObjectIds. objects = self.env.last_event.metadata['objects'] for idx in range(len(objects)): if objects[idx]['parentReceptacles'] is None: objects[idx]['parentReceptacles'] = [] for jdx in range(len(objects)): if (objects[jdx]['receptacleObjectIds'] is not None and objects[idx]['objectId'] in objects[jdx]['receptacleObjectIds']): objects[idx]['parentReceptacles'].append(objects[jdx]['objectId'])
def reset(self, scene_name=None, use_gt=True, seed=None): if scene_name is None: # Do half reset action_ind = self.local_random.randint( 0, constants.STEPS_AHEAD**2 - 1) action_x = action_ind % constants.STEPS_AHEAD - int( constants.STEPS_AHEAD / 2) action_z = int(action_ind / constants.STEPS_AHEAD) + 1 x_shift = 0 z_shift = 0 if self.pose[2] == 0: x_shift = action_x z_shift = action_z elif self.pose[2] == 1: x_shift = action_z z_shift = -action_x elif self.pose[2] == 2: x_shift = -action_x z_shift = -action_z elif self.pose[2] == 3: x_shift = -action_z z_shift = action_x action_x = self.pose[0] + x_shift action_z = self.pose[1] + z_shift self.end_point = (action_x, action_z, self.pose[2]) else: # Do full reset self.scene_name = scene_name grid_file = 'layouts/%s-layout.npy' % scene_name self.graph = graph_obj.Graph(grid_file, use_gt=use_gt) if seed is not None: self.local_random.seed(seed) lastActionSuccess = False self.bounds = [ self.graph.xMin, self.graph.yMin, self.graph.xMax - self.graph.xMin + 1, self.graph.yMax - self.graph.yMin + 1 ] while not lastActionSuccess: self.event = game_util.reset(self.env, self.scene_name) self.agent_height = self.event.metadata['agent']['position'][ 'y'] self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET self.event = self.env.random_initialize(seed) start_point = self.local_random.randint( 0, self.graph.points.shape[0] - 1) start_point = self.graph.points[start_point, :].copy() self.start_point = (start_point[0], start_point[1], self.local_random.randint(0, 3)) self.end_point = self.start_point while self.end_point[0] == self.start_point[ 0] and self.end_point[1] == self.start_point[1]: end_point = self.local_random.randint( 0, self.graph.points.shape[0] - 1) end_point = self.graph.points[end_point, :].copy() self.end_point = [ end_point[0], end_point[1], self.local_random.randint(0, 3) ] self.end_point[0] += self.local_random.randint( -constants.TERMINAL_CHECK_PADDING, constants.TERMINAL_CHECK_PADDING) self.end_point[1] += self.local_random.randint( -constants.TERMINAL_CHECK_PADDING, constants.TERMINAL_CHECK_PADDING) self.end_point = tuple(self.end_point) action = { 'action': 'TeleportFull', 'x': self.start_point[0] * constants.AGENT_STEP_SIZE, 'y': self.agent_height, 'z': self.start_point[1] * constants.AGENT_STEP_SIZE, 'rotateOnTeleport': True, 'rotation': self.start_point[2] * 90, 'horizon': 60 } self.event = self.env.step(action) lastActionSuccess = self.event.metadata['lastActionSuccess'] self.process_frame() self.board = None point_dists = np.sum(np.abs(self.graph.points - np.array(self.end_point[:2])), axis=1) dist_min = np.min(point_dists) self.is_possible_end_point = int(dist_min < 0.0001)
def create_dump(): time_str = py_util.get_time_str() prefix = 'questions/' if not os.path.exists(prefix + dataset_type + '/data_counting'): os.makedirs(prefix + dataset_type + '/data_counting') h5 = h5py.File( prefix + dataset_type + '/data_counting/Counting_Questions_' + time_str + '.h5', 'w') h5.create_dataset('questions/question', (num_record, 4), dtype=np.int32) print('Generating %d counting questions' % num_record) # Generate counting questions data_ind = 0 episode = Episode() scene_number = random.randint(0, len(scene_numbers) - 1) while data_ind < num_record: k = 0 scene_number += 1 scene_num = scene_numbers[scene_number % len(scene_numbers)] scene_name = 'FloorPlan%d' % scene_num episode.initialize_scene(scene_name) num_tries = 0 while num_tries < num_samples_per_scene: # randomly pick a pickable object in the scene object_class = random.choice(all_object_classes) question = CountQuestion( object_class ) # randomly generate a general counting question generated = [None] * (constants.MAX_COUNTING_ANSWER + 1) generated_counts = set() num_tries += 1 grid_file = 'layouts/%s-layout.npy' % scene_name xray_graph = graph_obj.Graph(grid_file, use_gt=True, construct_graph=False) scene_bounds = [ xray_graph.xMin, xray_graph.yMin, xray_graph.xMax - xray_graph.xMin + 1, xray_graph.yMax - xray_graph.yMin + 1 ] for i in range(100): if DEBUG: print('starting try ', i) scene_seed = random.randint(0, 999999999) episode.initialize_episode( scene_seed=scene_seed, max_num_repeats=constants.MAX_COUNTING_ANSWER + 1, remove_prob=0.5) answer = question.get_answer(episode) object_target = constants.OBJECT_CLASS_TO_ID[object_class] if answer > 0 and answer not in generated_counts: if DEBUG: print('target', str(question), object_target, answer) event = episode.event # Make sure findable try: objs = { obj['objectId']: obj for obj in event.metadata['objects'] if obj['objectType'] == object_class } xray_graph.memory[:, :, 1:] = 0 for obj in objs.values(): obj_point = game_util.get_object_point( obj, scene_bounds) xray_graph.memory[obj_point[1], obj_point[0], object_target + 1] = 1 start_graph = xray_graph.memory.copy() graph_points = xray_graph.points.copy() graph_points = graph_points[np.random.permutation( graph_points.shape[0]), :] num_checked_points = 0 point_ind = 0 # Initial check to make sure all objects are visible on the grid. while point_ind < len(graph_points): start_point = graph_points[point_ind] headings = np.random.permutation(4) for heading in headings: start_point = (start_point[0], start_point[1], heading) patch = xray_graph.get_graph_patch( start_point)[0][:, :, object_target + 1] if patch.max() > 0: point_ind = 0 xray_graph.update_graph( (np.zeros((constants.STEPS_AHEAD, constants.STEPS_AHEAD, 1)), 0), start_point, [object_target + 1]) point_ind += 1 if np.max(xray_graph.memory[:, :, object_target + 1]) > 0: if DEBUG: print('some points could not be reached') answer = None raise AssertionError xray_graph.memory = start_graph point_ind = 0 seen_objs = set() while point_ind < len(graph_points): start_point = graph_points[point_ind] headings = np.random.permutation(4) for heading in headings: start_point = (start_point[0], start_point[1], heading) patch = xray_graph.get_graph_patch( start_point)[0] if patch[:, :, object_target + 1].max() > 0: action = { 'action': 'TeleportFull', 'x': start_point[0] * constants.AGENT_STEP_SIZE, 'y': episode.agent_height, 'z': start_point[1] * constants.AGENT_STEP_SIZE, 'rotateOnTeleport': True, 'rotation': start_point[2] * 90, 'horizon': -30, } event = episode.env.step(action) num_checked_points += 1 if num_checked_points > 20: if DEBUG: print('timeout') answer = None raise AssertionError changed = False for jj in range(4): open_success = True opened_objects = set() parents = [ game_util.get_object( obj['parentReceptacle'], event.metadata) for obj in objs.values() ] openable_parents = [ parent for parent in parents if parent['visible'] and parent['openable'] and not parent['isopen'] ] while open_success: obj_list = list(objs.values()) for obj in obj_list: if obj['objectId'] in event.instance_detections2D: if game_util.check_object_size( event. instance_detections2D[ obj['objectId']] ): seen_objs.add( obj['objectId'] ) if DEBUG: print( 'seen', seen_objs) del objs[obj[ 'objectId']] changed = True num_checked_points = 0 if len(seen_objs ) == answer: raise AssertionError if len(openable_parents) > 0: action = { 'action': 'OpenObject' } game_util.set_open_close_object( action, event) event = episode.env.step( action) open_success = event.metadata[ 'lastActionSuccess'] if open_success: opened_objects.add( episode.env. last_action[ 'objectId']) else: open_success = False for opened in opened_objects: event = episode.env.step({ 'action': 'CloseObject', 'objectId': opened, 'forceVisible': True }) if not event.metadata[ 'lastActionSuccess']: answer = None raise AssertionError if jj < 3: event = episode.env.step( {'action': 'LookDown'}) if changed: point_ind = 0 num_checked_points = 0 xray_graph.memory[:, :, object_target + 1] = 0 for obj in objs.values(): obj_point = game_util.get_object_point( obj, scene_bounds) xray_graph.memory[ obj_point[1], obj_point[0], object_target + 1] = 1 point_ind += 1 if DEBUG: print('ran out of points') answer = None except AssertionError: if answer is not None: if DEBUG: print('success') pass print(str(question), object_target, answer) if answer is not None and answer < len( generated) and answer not in generated_counts: generated[answer] = [ scene_num, scene_seed, constants.OBJECT_CLASS_TO_ID[object_class], answer ] generated_counts.add(answer) print('\tcounts', sorted(list(generated_counts))) if len(generated_counts) == len(generated): for q in generated: if data_ind >= h5['questions/question'].shape[0]: num_tries = 2**32 break h5['questions/question'][data_ind, :] = np.array(q) data_ind += 1 k += 1 h5.flush() break print("# generated samples: {}".format(data_ind)) h5.close() episode.env.stop_unity()
def augment_traj(env, json_file): # load json data with open(json_file) as f: traj_data = json.load(f) # make directories root_dir = json_file.replace(TRAJ_DATA_JSON_FILENAME, "") # fresh images list traj_data['images'] = list() # scene setup scene_num = traj_data['scene']['scene_num'] object_poses = traj_data['scene']['object_poses'] object_toggles = traj_data['scene']['object_toggles'] dirty_and_empty = traj_data['scene']['dirty_and_empty'] # reset scene_name = 'FloorPlan%d' % scene_num env.reset(scene_name) env.restore_scene(object_poses, object_toggles, dirty_and_empty) print(scene_name) env.step(dict(traj_data['scene']['init_action'])) print("Task: %s" % (traj_data['turk_annotations']['anns'][0]['task_desc'])) # setup task env.set_task(traj_data, args, reward_type='dense') game_state = TextWorldTaskGameStateFullKnowledge(env) # reset game_state.receptacle_to_point = None game_state.task_target = None game_state.success = False # load nav graph game_state.gt_graph = graph_obj.Graph(use_gt=True, construct_graph=True, scene_id=scene_num) game_state.gt_graph.clear() game_state.agent_height = env.last_event.metadata['agent']['position']['y'] game_state.camera_height = game_state.agent_height + constants.CAMERA_HEIGHT_OFFSET points_source = 'gen/layouts/%s-openable.json' % scene_name with open(points_source, 'r') as f: openable_object_to_point = json.load(f) game_state.openable_object_to_point = openable_object_to_point pddl_params = traj_data['pddl_params'] game_state.object_target = constants.OBJECTS.index( pddl_params['object_target']) if pddl_params['object_target'] else None game_state.parent_target = constants.OBJECTS.index( pddl_params['parent_target']) if pddl_params['parent_target'] else None game_state.mrecep_target = constants.OBJECTS.index( pddl_params['mrecep_target']) if pddl_params['mrecep_target'] else None game_state.toggle_target = constants.OBJECTS.index( pddl_params['toggle_target']) if pddl_params['toggle_target'] else None game_state.task_target = (game_state.object_target, game_state.parent_target, game_state.toggle_target, game_state.mrecep_target) game_state.update_receptacle_nearest_points() pddl_str = game_state.state_to_pddl(traj_data) pddl_file = os.path.join(os.path.dirname(json_file), 'initial_state.pddl') with open(pddl_file, 'w') as f: f.write(pddl_str) print("Wrote to %s" % pddl_file) game_state.planner.process_pool.terminate()
def reset(self, seed=None, test_ind=None): if self.game_state.env is not None: self.game_state.reset(seed=seed, test_ind=test_ind) self.bounds = self.game_state.bounds self.end_points = np.zeros_like(self.game_state.graph.memory[:, :, 0]) for end_point in self.game_state.end_point: self.end_points[end_point[1] - self.game_state.graph.yMin, end_point[0] - self.game_state.graph.xMin] = 1 self.gru_state = np.zeros((1, constants.RL_GRU_SIZE)) self.pose = self.game_state.pose self.prev_pose = self.pose self.visited_poses = set() self.reward = 0 self.num_steps = 0 self.question_count = 0 self.num_invalid_actions = 0 self.prev_can_end = False dilation_kernel = np.ones( (constants.SCENE_PADDING * 2 + 1, constants.SCENE_PADDING)) free_space = self.game_state.xray_graph.memory.copy().squeeze() if len(free_space.shape) == 3: free_space = free_space[:, :, 0] free_space[free_space == graph_obj.MAX_WEIGHT] = 0 free_space[free_space > 1] = 1 self.dilation = np.zeros_like(free_space) for _ in range(2): self.dilation += scipy.ndimage.morphology.binary_dilation( free_space, structure=dilation_kernel).astype(np.int) dilation_kernel = np.rot90(dilation_kernel) self.dilation[self.dilation > 1] = 1 self.max_coverage = np.sum(self.dilation) # Rows are: # 0 - Map weights (not fed to decision network) # 1 and 2 - meshgrid # 3 - coverage # 4 - teleport locations # 5 - free space map # 6 - visited locations # 7+ - object location if constants.USE_NAVIGATION_AGENT: if self.nav_agent is not None: self.nav_agent.reset(self.game_state.scene_name) self.spatial_map = graph_obj.Graph('layouts/%s-layout.npy' % self.game_state.scene_name, use_gt=True, construct_graph=False) self.spatial_map.memory = np.concatenate( (np.zeros((self.bounds[3], self.bounds[2], 7)), self.game_state.graph.memory[:, :, 1:].copy()), axis=2) self.coverage = 0 self.terminal = False self.new_coverage = 0 self.last_meta_action = np.zeros(7) self.last_action_one_hot = np.zeros( self.network.pi.get_shape().as_list()[-1]) self.global_step_id += 1 self.update_spatial_map({'action': 'Initialize'}) # For drawing self.forward_pred = np.zeros((3, 3, 3)) self.next_memory_crops_rot = np.zeros((3, 3, 3))
def create_dump(): episode = Episode() count = 0 #for i in range(len(scene_numbers)): while count > -1: scene_name = 'FloorPlan%d' % scene_numbers[count] print('scene is %s' % scene_name) grid_file = 'layouts/%s-layout.npy' % scene_name graph = graph_obj.Graph(grid_file) #, use_gt=use_gt) print("graph: ", graph) scene_bounds = [ graph.xMin, graph.yMin, graph.xMax - graph.xMin + 1, graph.yMax - graph.yMin + 1 ] print("bounds: ", scene_bounds) episode.initialize_scene(scene_name) #scene_seed = random.randint(0, 999999999) #episode.initialize_episode(scene_seed=scene_seed) # randomly initialize the scene prefix = 'object_semantic_map/maps/' for index in range(1, 11): #memory = np.zeros((graph.yMax - graph.yMin + 1, graph.xMax - graph.xMin + 1, 1 + constants.NUM_CLASSES)) npy_path = os.path.join(prefix, str(index) + '.npy') memory = np.load(npy_path) print(memory.shape) """ for _ in range(SIM_TIMES): scene_seed = random.randint(0, 999999999) episode.initialize_episode(scene_seed=scene_seed) for obj in episode.get_objects(): if obj['objectType'] not in constants.OBJECTS: continue y, x = game_util.get_object_point(obj, scene_bounds) #print("object %s at " % obj['objectType'], game_util.get_object_point(obj, scene_bounds)) obj_id = constants.OBJECT_CLASS_TO_ID[obj['objectType']] memory[y][x][obj_id] += 1 memory = np.divide(memory, SIM_TIMES) """ print(memory.shape[:2]) #print (constants.OBJECTS[test_id]) #plt.figure(figsize=list(memory.shape[:2])) #plt.pcolor(memory[:,:,test_id],cmap=plt.get_cmap('Reds'), vmin=0.0, vmax=1.0)#,edgecolors='k', linewidths=1) #plt.colorbar() #print (memory[:,:,test_id]) #plt.show() fig, axs = plt.subplots(4, 7) #a1 for (i, j) in list(itertools.product("0123", "23456")): #print (i, j) obj_id = (int(i)) * 5 + int(j) - 1 #if obj_id > 20: break ax = axs[int(i), int(j)] pcm = ax.pcolor(memory[:, :, obj_id], cmap=plt.get_cmap('Reds'), vmin=-0, vmax=1) ax.set_title(constants.OBJECTS[obj_id]) ax.axis('off') gs = axs[0, 0].get_gridspec() for ax in axs[:, :2].flatten(): ax.remove() ax_frame = fig.add_subplot(gs[:, :2]) ax_frame.axis('off') new_frame = get_agent_map_data(episode.env) new_frame = new_frame[:, ::-1] new_frame = np.rot90(new_frame, -1) plt.imshow(new_frame) #fig.tight_layout() #fig.colorbar(pcm, ax=axs[:]) fig_path = os.path.join(prefix, str(index) + '.png') plt.savefig(fig_path) """ objs = [obj['objectType'] for obj in episode.get_objects()] objjs = [obj['objectId'] for obj in episode.get_objects()] objjspos = [obj['position'] for obj in episode.get_objects()] for ii in range(len(episode.get_objects())): print("object%d at " % ii, game_util.get_object_point(episode.get_objects()[ii], scene_bounds)) print("objs: ", objs) print("objjs: ", objjs) print("objjs: ", len(objjs)) print("objjspos: ", objjspos) print("objjspos: ", len(objjspos)) """ #event = episode.get_env_top_view() #print (event.metadata) #image = drawing.subplot() break