def test_random_initialize_speed(env): print('Reset/Initialize speed') t_start = time.time() time_count = 0 for scene in range(1,31): time_count += 1 game_util.reset(env, scene) event = env.random_initialize(random_seed=0) t_end = time.time() total_time = t_end - t_start print('Total time %.3f' % (total_time)) print('Mean time %.3f' % (total_time / time_count)) print('FPS = %.3f' % (time_count / total_time)) print('')
def initialize_episode(self, scene_seed=None, agent_seed=None, max_num_repeats=10, remove_prob=0.25): """Initializes environment with given scene and random seed.""" # Reset the scene with some random seed. if scene_seed is None: scene_seed = random.randint(0, 999999999) if agent_seed is None: agent_seed = random.randint(0, 999999999) self.event = game_util.reset(self.env, self.scene_name, render_depth_image=False, render_class_image=False, render_object_image=True) self.event = self.env.random_initialize( random_seed=scene_seed, max_num_repeats=max_num_repeats, remove_prob=remove_prob) self.agent_height = self.event.metadata['agent']['position']['y'] self.is_initialized = True return scene_seed, agent_seed
def test_movement_speed(env): print('Movement speed') time_count = 0 total_time = 0 for scene in range(1,31): game_util.reset(env, scene) event = env.random_initialize(random_seed=0) t_start = time.time() for jj in range(10): event = env.step({'action': 'MoveAhead'}) event = env.step({'action': 'RotateRight'}) time_count += 2 total_time += time.time() - t_start print('Total time %.3f' % (total_time)) print('Mean time %.3f' % (total_time / time_count)) print('FPS = %.3f' % (time_count / total_time)) print('')
def get_env_info(self): """Get env specific information.""" event = game_util.reset(self.env, self.scene_name, render_depth_image=False, render_class_image=False, render_object_image=True) self.object_id_to_object_class = { obj['objectId']: obj['objectType'] for obj in event.metadata['objects'] } self.pickable_object_classes = sorted( list( set([ obj['objectType'] for obj in event.metadata['objects'] if obj['pickupable'] and obj['objectType'] != 'MiscObject' ]))) print('# Pickable object_classes:', len(self.pickable_object_classes)) # Find all receptacles. self.receptacles = sorted([ obj['objectId'] for obj in event.metadata['objects'] if obj['receptacle'] ]) print('# Receptacles:', len(self.receptacles)) # Find all receptacle classes. self.receptacle_classes = list( set([item.split('|')[0] for item in self.receptacles])) print('# Receptacle classes:', len(self.receptacle_classes)) # Find all openable receptacles. self.openable_receptacles = sorted([ obj['objectId'] for obj in event.metadata['objects'] if obj['receptacle'] and obj['openable'] ]) print('# Openable Receptacles:', len(self.openable_receptacles)) # Find all openable receptacle classes. self.openable_receptacle_classes = list( set([item.split('|')[0] for item in self.openable_receptacles])) print('# Openable object_classes:', len(self.openable_receptacle_classes)) self.agent_height = event.metadata['agent']['position']['y']
def reset(self, seed=None, test_ind=None): self.board = None self.seen_object = False self.terminal = False self.opened_receptacles = set() self.closed_receptacles = set() self.seen_obj1 = set() self.seen_obj2 = set() self.visited_locations = set() self.can_end = False if seed is not None: self.local_random.seed(seed) # Do equal number of each question type in train. question_type_ind = self.local_random.sample( constants.USED_QUESTION_TYPES, 1)[0] # get random row if test_ind is not None: question_row, question_type_ind = test_ind question_type = self.question_types[question_type_ind] question_data = self.test_datasets[question_type_ind][ question_row, :] test_ind = (question_row, question_type_ind) else: question_type = self.question_types[question_type_ind] question_row = self.local_random.randint( 0, len(self.datasets[question_type_ind]) - 1) question_data = self.datasets[question_type_ind][question_row, :] container_ind = None if question_type_ind == 0 or question_type_ind == 1: scene_num, scene_seed, object_ind, answer = question_data self.question_target = object_ind if question_type_ind == 0: answer = bool(answer) elif question_type_ind == 2: scene_num, scene_seed, object_ind, container_ind, answer = question_data answer = bool(answer) self.question_target = (object_ind, container_ind) else: raise Exception('No question type found for type %d' % question_type_ind) self.scene_seed = scene_seed self.scene_num = scene_num self.object_target = object_ind self.parent_target = container_ind self.container_target = np.zeros(constants.NUM_CLASSES) self.direction_target = np.zeros(4) if container_ind is not None: self.container_target[container_ind] = 1 self.question_type_ind = question_type_ind self.scene_name = 'FloorPlan%d' % scene_num grid_file = 'layouts/%s-layout.npy' % self.scene_name self.graph = graph_obj.Graph(grid_file, use_gt=False) self.xray_graph = graph_obj.Graph(grid_file, use_gt=True) self.bounds = [ self.graph.xMin, self.graph.yMin, self.graph.xMax - self.graph.xMin + 1, self.graph.yMax - self.graph.yMin + 1 ] max_num_repeats = 1 remove_prob = 0.5 if question_type == 'existence': max_num_repeats = 10 remove_prob = 0.25 elif question_type == 'counting': max_num_repeats = constants.MAX_COUNTING_ANSWER + 1 remove_prob = 0.5 elif question_type == 'contains': max_num_repeats = 10 remove_prob = 0.25 self.event = game_util.reset(self.env, self.scene_name) self.agent_height = self.event.metadata['agent']['position']['y'] self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET self.event = self.env.random_initialize( self.scene_seed, max_num_repeats=max_num_repeats, remove_prob=remove_prob) print('Type:', question_type, 'Row: ', question_row, 'Scene', self.scene_name, 'seed', scene_seed) print( 'Question:', game_util.get_question_str(question_type_ind, object_ind, container_ind)) if self.question_type_ind == 2: print('Answer:', constants.OBJECTS[object_ind], 'in', constants.OBJECTS[container_ind], 'is', answer) else: print('Answer:', constants.OBJECTS[object_ind], 'is', answer) self.answer = answer # Verify answer if self.question_type_ind == 0: objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind], self.event.metadata) computed_answer = len(objs) > 0 requires_interaction = True for obj in objs: parent = obj['parentReceptacle'].split('|')[0] if parent not in {'Fridge', 'Cabinet', 'Microwave'}: requires_interaction = False break elif self.question_type_ind == 1: objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind], self.event.metadata) computed_answer = len(objs) requires_interaction = True elif self.question_type_ind == 2: objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind], self.event.metadata) if len(objs) == 0: computed_answer = False requires_interaction = constants.OBJECTS[ self.question_target[1]] in { 'Fridge', 'Cabinet', 'Microwave' } else: obj = objs[0] computed_answer = False for obj in objs: requires_interaction = True parent = obj['parentReceptacle'].split('|')[0] if parent in constants.OBJECT_CLASS_TO_ID: parent_ind = constants.OBJECT_CLASS_TO_ID[parent] computed_answer = parent_ind == self.question_target[1] if computed_answer: if parent not in { 'Fridge', 'Cabinet', 'Microwave' }: requires_interaction = False break else: computed_answer = False self.requires_interaction = requires_interaction try: assert self.answer == computed_answer, 'Answer does not match scene metadata' except AssertionError: print('Type:', question_type, 'Row: ', question_row, 'Scene', self.scene_name, 'seed', scene_seed) print('Answer', computed_answer, 'does not match expected value', self.answer, ', did randomization process change?') pdb.set_trace() self.answer = computed_answer if constants.NUM_CLASSES > 1: self.hidden_items = set() objects = self.event.metadata['objects'] for obj in objects: if obj['receptacle'] and obj['openable'] and not obj['isopen']: for inside_obj in obj['receptacleObjectIds']: self.hidden_items.add(inside_obj) objects = self.event.metadata['objects'] for obj in objects: if obj['objectType'] not in constants.OBJECT_CLASS_TO_ID: continue obj_bounds = game_util.get_object_bounds(obj, self.bounds) self.xray_graph.memory[ obj_bounds[1]:obj_bounds[3], obj_bounds[0]:obj_bounds[2], constants.OBJECT_CLASS_TO_ID[obj['objectType']] + 1] = 1 start_point = self.local_random.randint(0, self.graph.points.shape[0] - 1) start_point = self.graph.points[start_point, :].copy() self.start_point = (start_point[0], start_point[1], self.local_random.randint(0, 3)) action = { 'action': 'TeleportFull', 'x': self.start_point[0] * constants.AGENT_STEP_SIZE, 'y': self.agent_height, 'z': self.start_point[1] * constants.AGENT_STEP_SIZE, 'rotateOnTeleport': True, 'rotation': self.start_point[2] * 90, 'horizon': 30, } self.event = self.env.step(action) self.process_frame() self.reward = 0 self.end_point = []
def reset(self, scene_name=None, use_gt=True, seed=None, config_filename=""): if scene_name is None: # Do half reset action_ind = self.local_random.randint( 0, constants.STEPS_AHEAD**2 - 1) action_x = action_ind % constants.STEPS_AHEAD - int( constants.STEPS_AHEAD / 2) action_z = int(action_ind / constants.STEPS_AHEAD) + 1 x_shift = 0 z_shift = 0 if self.pose[2] == 0: x_shift = action_x z_shift = action_z elif self.pose[2] == 1: x_shift = action_z z_shift = -action_x elif self.pose[2] == 2: x_shift = -action_x z_shift = -action_z elif self.pose[2] == 3: x_shift = -action_z z_shift = action_x action_x = self.pose[0] + x_shift action_z = self.pose[1] + z_shift self.end_point = (action_x, action_z, self.pose[2]) #print ("in the game state reset end point is : ", self.end_point) else: # Do full reset self.scene_name = scene_name #print ("Full reset - in the first time of load") grid_file = 'layouts/%s-layout_%s.npy' % ( scene_name, str(constants.AGENT_STEP_SIZE)) self.graph = graph_obj.Graph(grid_file, self.action_util, use_gt=use_gt) if seed is not None: self.local_random.seed(seed) lastActionSuccess = False self.discovered_explored = {} self.discovered_objects = [] self.bounds = [ self.graph.xMin, self.graph.yMin, self.graph.xMax - self.graph.xMin + 1, self.graph.yMax - self.graph.yMin + 1 ] ''' while not lastActionSuccess: print ("In the while loop") self.event = game_util.reset(self.env, self.scene_name) self.event = self.event.events[0] self.agent_height = self.event.metadata['agent']['position']['y'] self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET #self.event = self.env.random_initialize(seed) #self.start_point = [] start_point_2 = [] #start_point_2.append(int(self.event.metadata['agent']['position']['x']/constants.AGENT_STEP_SIZE) -self.graph.xMin +1) #start_point_2.append(int(self.event.metadata['agent']['position']['z']/constants.AGENT_STEP_SIZE) - self.graph.yMin +1 ) start_point_2.append(int(self.event.metadata['agent']['position']['x']/constants.AGENT_STEP_SIZE) ) start_point_2.append(int(self.event.metadata['agent']['position']['z']/constants.AGENT_STEP_SIZE) ) start_point_2.append(int(self.event.metadata['agent']['rotation']['y']/90)) print(start_point_2) print ("starting horizon", self.event.metadata['agent']['cameraHorizon']) self.start_point = start_point_2[:] #start_point = self.local_random.randint(0, self.graph.points.shape[0] - 1) #start_point = self.graph.points[start_point, :].copy() #self.start_point = (start_point[0], start_point[1], self.local_random.randint(0, 3)) self.end_point = self.start_point #print ("B4 the while loop for assigning end point", self.start_point) while self.end_point[0] == self.start_point[0] and self.end_point[1] == self.start_point[1]: #end_point = self.local_random.randint(0, self.graph.points.shape[0] - 1) #end_point = self.graph.points[end_point, :].copy() #self.end_point = [end_point[0], end_point[1], self.local_random.randint(0, 3)] #self.end_point[0] += self.local_random.randint(-constants.TERMINAL_CHECK_PADDING, constants.TERMINAL_CHECK_PADDING) #self.end_point[1] += self.local_random.randint(-constants.TERMINAL_CHECK_PADDING, constants.TERMINAL_CHECK_PADDING) #self.end_point = tuple(self.end_point) self.end_point = (self.end_point[0]-1 , self.end_point[1]-1 , self.end_point[2]) print ("In the while loop for assigning end point",self.end_point) action = {'action': 'TeleportFull', 'x': self.start_point[0] * constants.AGENT_STEP_SIZE, 'y': self.agent_height, 'z': self.start_point[1] * constants.AGENT_STEP_SIZE, 'rotateOnTeleport': True, 'rotation': self.start_point[2] * 90, 'horizon': 0} #'horizon': 60} ''' while True: self.event = game_util.reset(self.env, self.scene_name, config_filename) #self.event = self.event.events[0] print("type of event 2 : ", type(self.event)) lastActionSuccess = self.event.return_status break start_point_2 = [] ''' self.agent_height = self.event.metadata['agent']['position']['y'] self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET start_point_2.append(int(self.event.metadata['agent']['position']['x']/constants.AGENT_STEP_SIZE) ) start_point_2.append(int(self.event.metadata['agent']['position']['z']/constants.AGENT_STEP_SIZE) ) start_point_2.append(int(self.event.metadata['agent']['rotation']['y']/90)) ''' print("rotation ", self.event.rotation) rotation_angle = -(self.event.rotation % 90) action = "RotateLook, rotation=%d" % int(rotation_angle) self.goal = self.event.goal #action = {'action':"Pass"} #print ("z before turning ", self.event.position['z']/constants.AGENT_STEP_SIZE) #print ("z before turning ", self.event.position['z']) self.event = self.env.step(action) #print ("z after turning ", self.event.position['z']/constants.AGENT_STEP_SIZE) #print ("z after turning ", self.event.position['z']) self.agent_height = self.event.position['y'] self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET start_point_2.append( math.floor(self.event.position['x'] / constants.AGENT_STEP_SIZE)) start_point_2.append( math.floor(self.event.position['z'] / constants.AGENT_STEP_SIZE)) start_point_2.append(int(self.event.rotation / 90)) #print(start_point_2) self.start_point = start_point_2[:] self.end_point = self.start_point[:] self.end_point = (self.end_point[0], self.end_point[1], (self.end_point[2] + 1) % 4) #print ("end point = ", self.end_point) #print ("start point = ", self.start_point) #print ("last action success", self.event['lastActionSuccess']) #print (self.event) #print ("last action success", self.event.lastActionSuccess) #print ("number of objects : ",len(self.event.metadata['objects'])) #action = {"action": "RotateRight", 'rotatio#n':90} #print ("type of event 3 : ", type(self.event)) #action = {"action": "RotateLeft", 'rotation':90} #action = "RotateLook, rotation=-90" #self.event = self.env.step(action) #print ("type of event 4 : ", type(self.event)) #self.event = self.event.events[0] #mcs_output = wrap_output(self.event) #lastActionSuccess = self.event.metadata['lastActionSuccess'] lastActionSuccess = self.event.return_status print("last action sucess", lastActionSuccess) self.process_frame() #print (self.get_pose()) #self.pose = game_util.get_pose(self.event) print("current pose = ", self.pose) self.board = None #point_dists = np.sum(np.abs(self.graph.points - np.array(self.end_point[:2])), axis=1) #print ("jsut b4 end of function") #dist_min = np.min(point_dists) #self.is_possible_end_point = int(dist_min < 0.0001) print("end of reset in game state function")
def reset(self, seed=None, test_ind=None): if seed is not None: self.local_random.seed(seed) question_row, question_type_ind = test_ind question_type = self.question_types[question_type_ind] question_data = self.test_datasets[question_type_ind][ question_row % len(self.test_datasets[question_type_ind])] scene_num, scene_seed, question_str, answer = question_data[1:5] self.scene_seed = int(scene_seed) self.scene_num = int(scene_num) self.question_str = question_str self.question_type_ind = question_type_ind self.scene_name = 'FloorPlan%d' % self.scene_num grid_file = 'layouts/%s-layout.npy' % self.scene_name self.points = (np.load(grid_file) * 1.0 / constants.AGENT_STEP_SIZE).astype(int) max_num_repeats = 1 remove_prob = 0.5 if question_type == 'existence': max_num_repeats = 10 remove_prob = 0.25 elif question_type == 'counting': max_num_repeats = constants.MAX_COUNTING_ANSWER + 1 remove_prob = 0.5 elif question_type == 'contains': max_num_repeats = 10 remove_prob = 0.25 self.event = game_util.reset(self.env, self.scene_name, render_image=True, render_depth_image=True, render_class_image=True, render_object_image=True) self.agent_height = self.event.metadata['agent']['position']['y'] self.event = self.env.random_initialize( self.scene_seed, max_num_repeats=max_num_repeats, remove_prob=remove_prob) print('Question: %s' % self.question_str) if answer == 'True': self.answer = True elif answer == 'False': self.answer = True else: self.answer = int(answer) start_point = self.local_random.randint(0, self.points.shape[0] - 1) start_point = self.points[start_point, :].copy() self.start_point = (start_point[0], start_point[1], self.local_random.randint(0, 3)) action = { 'action': 'TeleportFull', 'x': self.start_point[0] * constants.AGENT_STEP_SIZE, 'y': self.agent_height, 'z': self.start_point[1] * constants.AGENT_STEP_SIZE, 'rotateOnTeleport': True, 'rotation': self.start_point[2] * 90, 'horizon': 30, } self.event = self.env.step(action) self.process_frame()
def reset(self, scene_name=None, use_gt=True, seed=None): if scene_name is None: # Do half reset action_ind = self.local_random.randint( 0, constants.STEPS_AHEAD**2 - 1) action_x = action_ind % constants.STEPS_AHEAD - int( constants.STEPS_AHEAD / 2) action_z = int(action_ind / constants.STEPS_AHEAD) + 1 x_shift = 0 z_shift = 0 if self.pose[2] == 0: x_shift = action_x z_shift = action_z elif self.pose[2] == 1: x_shift = action_z z_shift = -action_x elif self.pose[2] == 2: x_shift = -action_x z_shift = -action_z elif self.pose[2] == 3: x_shift = -action_z z_shift = action_x action_x = self.pose[0] + x_shift action_z = self.pose[1] + z_shift self.end_point = (action_x, action_z, self.pose[2]) else: # Do full reset self.scene_name = scene_name grid_file = 'layouts/%s-layout.npy' % scene_name self.graph = graph_obj.Graph(grid_file, use_gt=use_gt) if seed is not None: self.local_random.seed(seed) lastActionSuccess = False self.bounds = [ self.graph.xMin, self.graph.yMin, self.graph.xMax - self.graph.xMin + 1, self.graph.yMax - self.graph.yMin + 1 ] while not lastActionSuccess: self.event = game_util.reset(self.env, self.scene_name) self.agent_height = self.event.metadata['agent']['position'][ 'y'] self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET self.event = self.env.random_initialize(seed) start_point = self.local_random.randint( 0, self.graph.points.shape[0] - 1) start_point = self.graph.points[start_point, :].copy() self.start_point = (start_point[0], start_point[1], self.local_random.randint(0, 3)) self.end_point = self.start_point while self.end_point[0] == self.start_point[ 0] and self.end_point[1] == self.start_point[1]: end_point = self.local_random.randint( 0, self.graph.points.shape[0] - 1) end_point = self.graph.points[end_point, :].copy() self.end_point = [ end_point[0], end_point[1], self.local_random.randint(0, 3) ] self.end_point[0] += self.local_random.randint( -constants.TERMINAL_CHECK_PADDING, constants.TERMINAL_CHECK_PADDING) self.end_point[1] += self.local_random.randint( -constants.TERMINAL_CHECK_PADDING, constants.TERMINAL_CHECK_PADDING) self.end_point = tuple(self.end_point) action = { 'action': 'TeleportFull', 'x': self.start_point[0] * constants.AGENT_STEP_SIZE, 'y': self.agent_height, 'z': self.start_point[1] * constants.AGENT_STEP_SIZE, 'rotateOnTeleport': True, 'rotation': self.start_point[2] * 90, 'horizon': 60 } self.event = self.env.step(action) lastActionSuccess = self.event.metadata['lastActionSuccess'] self.process_frame() self.board = None point_dists = np.sum(np.abs(self.graph.points - np.array(self.end_point[:2])), axis=1) dist_min = np.min(point_dists) self.is_possible_end_point = int(dist_min < 0.0001)
def get_obj(env, open_test_objs, reachable_points, agent_height, scene_name, good_obj_point): # Reset the scene to put all the objects back where they started. game_util.reset(env, scene_name, render_image=False, render_depth_image=False, render_class_image=False, render_object_image=True) if good_obj_point is not None: search_points = {good_obj_point[0]} else: search_points = reachable_points for point in search_points: for rotation in range(4): if good_obj_point is not None and rotation != good_obj_point[1]: continue for horizon in [-30, 0, 30]: if good_obj_point is not None and horizon != good_obj_point[2]: continue action = {'action': 'TeleportFull', 'x': point[0], 'y': agent_height, 'z': point[1], 'rotateOnTeleport': True, 'rotation': rotation * 90, 'horizon': horizon } event = env.step(action) if event.metadata['lastActionSuccess']: # Close everything. for obj in event.metadata['objects']: if (obj['visible'] and obj['objectId'] and obj['isOpen'] and obj['objectType'] in constants.VAL_RECEPTACLE_OBJECTS): action = {'action': 'CloseObject', 'objectId': obj['objectId']} event = env.step(action) # Try to pick up a valid inv object. for obj in event.metadata['objects']: if obj['visible'] and obj['objectType'] in open_test_objs: action = {'action': 'PickupObject', 'objectId': obj['objectId']} event = env.step(action) if event.metadata['lastActionSuccess']: good_obj_point = (point, rotation, horizon) return good_obj_point # Open everything. for open_obj in event.metadata['objects']: if (open_obj['visible'] and open_obj['objectId'] and open_obj['openable'] and not open_obj['pickupable'] and open_obj['objectType'] in constants.VAL_RECEPTACLE_OBJECTS): action = {'action': 'OpenObject', 'objectId': open_obj['objectId']} event = env.step(action) if event.metadata['lastActionSuccess']: # Try to pick up a valid inv object. for obj in event.metadata['objects']: if obj['visible'] and obj['objectType'] in open_test_objs: action = {'action': 'PickupObject', 'objectId': obj['objectId']} event = env.step(action) if event.metadata['lastActionSuccess']: good_obj_point = (point, rotation, horizon) return good_obj_point # Close open_obj. action = {'action': 'CloseObject', 'objectId': open_obj['objectId']} event = env.step(action) return None
def run(): print(all_scene_numbers) # create env and agent env = game_util.create_env(build_path=constants.BUILD_PATH, quality='Low') while len(all_scene_numbers) > 0: lock.acquire() scene_num = all_scene_numbers.pop() lock.release() fn = os.path.join('layouts', ('FloorPlan%d-layout.npy') % scene_num) if os.path.isfile(fn): print("file %s already exists; skipping this floorplan" % fn) continue openable_json_file = os.path.join('layouts', ('FloorPlan%d-openable.json') % scene_num) scene_objs_json_file = os.path.join('layouts', ('FloorPlan%d-objects.json') % scene_num) scene_name = ('FloorPlan%d') % scene_num print('Running ' + scene_name) event = game_util.reset(env, scene_name, render_image=False, render_depth_image=False, render_class_image=False, render_object_image=True) agent_height = event.metadata['agent']['position']['y'] scene_objs = list(set([obj['objectType'] for obj in event.metadata['objects']])) with open(scene_objs_json_file, 'w') as sof: json.dump(scene_objs, sof, sort_keys=True, indent=4) # Get all the reachable points through Unity for this step size. event = env.step(dict(action='GetReachablePositions', gridSize=constants.AGENT_STEP_SIZE / constants.RECORD_SMOOTHING_FACTOR)) if event.metadata['actionReturn'] is None: print("ERROR: scene %d 'GetReachablePositions' returns None" % scene_num) else: reachable_points = set() for point in event.metadata['actionReturn']: reachable_points.add((point['x'], point['z'])) print("scene %d got %d reachable points, now checking" % (scene_num, len(reachable_points))) # Pick up a small object to use in testing whether points are good for openable objects. open_test_objs = {'ButterKnife', 'CD', 'CellPhone', 'Cloth', 'CreditCard', 'DishSponge', 'Fork', 'KeyChain', 'Pen', 'Pencil', 'SoapBar', 'Spoon', 'Watch'} good_obj_point = None good_obj_point = get_obj(env, open_test_objs, reachable_points, agent_height, scene_name, good_obj_point) best_open_point = {} # map from object names to the best point from which they can be successfully opened best_sem_coverage = {} # number of pixels in the semantic map of the receptacle at the existing best openpt checked_points = set() scene_receptacles = set() for point in reachable_points: point_is_valid = True action = {'action': 'TeleportFull', 'x': point[0], 'y': agent_height, 'z': point[1], } event = env.step(action) if event.metadata['lastActionSuccess']: for horizon in [-30, 0, 30]: action = {'action': 'TeleportFull', 'x': point[0], 'y': agent_height, 'z': point[1], 'rotateOnTeleport': True, 'rotation': 0, 'horizon': horizon } event = env.step(action) if not event.metadata['lastActionSuccess']: point_is_valid = False break for rotation in range(3): action = {'action': 'RotateLeft'} event = env.step(action) if not event.metadata['lastActionSuccess']: point_is_valid = False break if not point_is_valid: break if point_is_valid: checked_points.add(point) else: continue # Check whether we can open objects from here in any direction with any tilt. for rotation in range(4): # First try up, then down, then return to the horizon before moving again. for horizon in [-30, 0, 30]: action = {'action': 'TeleportFull', 'x': point[0], 'y': agent_height, 'z': point[1], 'rotateOnTeleport': True, 'rotation': rotation * 90, 'horizon': horizon } event = env.step(action) for obj in event.metadata['objects']: if (obj['visible'] and obj['objectId'] and obj['receptacle'] and not obj['pickupable'] and obj['objectType'] in constants.VAL_RECEPTACLE_OBJECTS): obj_name = obj['objectId'] obj_point = (obj['position']['x'], obj['position']['y']) scene_receptacles.add(obj_name) # Go ahead and attempt to close the object from this position if it's open. if obj['openable'] and obj['isOpen']: close_action = {'action': 'CloseObject', 'objectId': obj['objectId']} event = env.step(close_action) point_to_recep = np.linalg.norm(np.array(point) - np.array(obj_point)) if len(env.last_event.metadata['inventoryObjects']) > 0: inv_obj = env.last_event.metadata['inventoryObjects'][0]['objectId'] else: inv_obj = None # Heuristic implemented in task_game_state has agent 0.5 or farther in agent space. heuristic_far_enough_from_recep = 0.5 < point_to_recep # Ensure this point affords a larger view according to the semantic segmentation # of the receptacle than the existing. point_sem_coverage = get_mask_of_obj(env, obj['objectId']) if point_sem_coverage is None: use_sem_heuristic = False better_sem_covereage = False else: use_sem_heuristic = True better_sem_covereage = (obj_name not in best_sem_coverage or best_sem_coverage[obj_name] is None or point_sem_coverage > best_sem_coverage[obj_name]) # Ensure that this point is farther away than our existing best candidate. # We'd like to open each receptacle from as far away as possible while retaining # the ability to pick/place from it. farther_than_existing_good_point = (obj_name not in best_open_point or point_to_recep > np.linalg.norm( np.array(point) - np.array(best_open_point[obj_name][:2]))) # If we don't have an inventory object, though, we'll fall back to the heuristic # of being able to open/close as _close_ as possible. closer_than_existing_good_point = (obj_name not in best_open_point or point_to_recep < np.linalg.norm( np.array(point) - np.array(best_open_point[obj_name][:2]))) # Semantic segmentation heuristic. if ((use_sem_heuristic and heuristic_far_enough_from_recep and better_sem_covereage) or (not use_sem_heuristic and # Distance heuristics. (heuristic_far_enough_from_recep and (inv_obj and farther_than_existing_good_point) or (not inv_obj and closer_than_existing_good_point)))): if obj['openable']: action = {'action': 'OpenObject', 'objectId': obj['objectId']} event = env.step(action) if not obj['openable'] or event.metadata['lastActionSuccess']: # We can open the object, so try placing our small inventory obj inside. # If it can be placed inside and retrieved, then this is a safe point. action = {'action': 'PutObject', 'objectId': inv_obj, 'receptacleObjectId': obj['objectId'], 'forceAction': True, 'placeStationary': True} if inv_obj: event = env.step(action) if inv_obj is None or event.metadata['lastActionSuccess']: action = {'action': 'PickupObject', 'objectId': inv_obj} if inv_obj: event = env.step(action) if inv_obj is None or event.metadata['lastActionSuccess']: # Finally, ensure we can also close the receptacle. if obj['openable']: action = {'action': 'CloseObject', 'objectId': obj['objectId']} event = env.step(action) if not obj['openable'] or event.metadata['lastActionSuccess']: # We can put/pick our inv object into the receptacle from here. # We have already ensured this point is farther than any # existing best, so this is the new best. best_open_point[obj_name] = [point[0], point[1], rotation * 90, horizon] best_sem_coverage[obj_name] = point_sem_coverage # We could not retrieve our inv object, so we need to go get another one else: good_obj_point = get_obj(env, open_test_objs, reachable_points, agent_height, scene_name, good_obj_point) action = {'action': 'TeleportFull', 'x': point[0], 'y': agent_height, 'z': point[1], 'rotateOnTeleport': True, 'rotation': rotation * 90, 'horizon': horizon } event = env.step(action) # Regardless of what happened up there, try to close the receptacle again if # it remained open. if obj['isOpen']: action = {'action': 'CloseObject', 'objectId': obj['objectId']} event = env.step(action) essential_objs = [] if scene_num in constants.SCENE_TYPE["Kitchen"]: essential_objs.extend(["Microwave", "Fridge"]) for obj in essential_objs: if not np.any([obj in obj_key for obj_key in best_open_point]): print("WARNING: Essential object %s has no open points in scene %d" % (obj, scene_num)) print("scene %d found open/pick/place/close positions for %d/%d receptacle objects" % (scene_num, len(best_open_point), len(scene_receptacles))) with open(openable_json_file, 'w') as f: json.dump(best_open_point, f, sort_keys=True, indent=4) print("scene %d reachable %d, checked %d; taking intersection" % (scene_num, len(reachable_points), len(checked_points))) points = np.array(list(checked_points))[:, :2] points = points[np.lexsort((points[:, 0], points[:, 1])), :] np.save(fn, points) env.stop() print('Done')
def reset(self, scene_name=None, use_gt=True, seed=None, config_filename= "",event=None): if scene_name is None: # Do half reset action_ind = self.local_random.randint(0, constants.STEPS_AHEAD ** 2 - 1) action_x = action_ind % constants.STEPS_AHEAD - int(constants.STEPS_AHEAD / 2) action_z = int(action_ind / constants.STEPS_AHEAD) + 1 x_shift = 0 z_shift = 0 if self.pose[2] == 0: x_shift = action_x z_shift = action_z elif self.pose[2] == 1: x_shift = action_z z_shift = -action_x elif self.pose[2] == 2: x_shift = -action_x z_shift = -action_z elif self.pose[2] == 3: x_shift = -action_z z_shift = action_x action_x = self.pose[0] + x_shift action_z = self.pose[1] + z_shift self.end_point = (action_x, action_z, self.pose[2]) #print ("in the game state reset end point is : ", self.end_point) else: # Do full reset #self.world_poly = fov.FieldOfView([0, 0, 0], 0, []) self.world_poly = sp.Polygon() self.goals_found = False self.scene_name = scene_name self.number_actions = 0 self.id_goal_in_hand = None #print ("Full reset - in the first time of load") #grid_file = 'layouts/%s-layout_%s.npy' % (scene_name,str(constants.AGENT_STEP_SIZE)) self.graph = None self.goal_in_hand = False if seed is not None: self.local_random.seed(seed) lastActionSuccess = False self.discovered_explored = {} self.discovered_objects = [] self.bounds = None #while True : #self.event = self.event.events[0] if event != None : self.event = event else : self.event = game_util.reset(self.env, self.scene_name,config_filename) self.goals = [] for key,value in self.event.goal.metadata.items(): if key == "target" or key == "target_1" or key == "target_2": self.goals.append(self.event.goal.metadata[key]["id"]) for obj in self.event.object_list: if obj.uuid not in self.discovered_explored: print("uuid : ", obj.uuid) self.discovered_explored[obj.uuid] = {0: obj.position} self.discovered_objects.append(obj.__dict__) self.discovered_objects[-1]['locationParent'] = None self.discovered_objects[-1]['explored'] = 0 self.discovered_objects[-1]['openable'] = None #self.discovered_objects[-1]['agent_position'] = None self.add_obstacle_func(self.event) #print ("type of event 2 : ", type(self.event)) lastActionSuccess = self.event.return_status #break self.process_frame() self.board = None