def get_true_contain_question(episode, things, receptacles): receptacles = list(receptacles) random.shuffle(receptacles) for parent in receptacles: parents = game_util.get_objects_of_type(parent, episode.event.metadata) random.shuffle(parents) for parent in parents: if len(parent['pivotSimObjs']) == 0: continue obj = random.choice(parent['pivotSimObjs']) obj_type = obj['objectId'].split('|')[0] if obj_type in things: return ExistenceQuestion(obj_type, parent['objectType']) return None
def step(self, action_or_ind): self.reward = -0.01 action, teleport_failure, should_fail = self.get_action(action_or_ind) t_start = time.time() if should_fail or teleport_failure: self.event.metadata['lastActionSuccess'] = False else: if action[ 'action'] != 'Teleport' or not constants.USE_NAVIGATION_AGENT: self.event = self.env.step(action) else: # Action is teleport and I should do low level navigation. pass new_pose = game_util.get_pose(self.event) point_dists = np.sum(np.abs(self.graph.points - np.array(new_pose)[:2]), axis=1) if np.min(point_dists) > 0.0001: print('Point teleport failure') closest_point = self.graph.points[np.argmin(point_dists)] self.event = self.env.step({ 'action': 'Teleport', 'x': closest_point[0] * constants.AGENT_STEP_SIZE, 'y': self.agent_height, 'z': closest_point[1] * constants.AGENT_STEP_SIZE, 'rotateOnTeleport': True, 'rotation': self.pose[2] * 90, }) else: closest_point = np.argmin(point_dists) if closest_point not in self.visited_locations: self.visited_locations.add(closest_point) self.times[2, 0] += time.time() - t_start self.times[2, 1] += 1 if self.times[2, 1] % 100 == 0: print('env step time %.3f' % (self.times[2, 0] / self.times[2, 1])) if self.event.metadata['lastActionSuccess']: self.process_frame() if action['action'] == 'OpenObject': if self.question_type_ind == 2 and action['objectId'].split( '|')[0] != constants.OBJECTS[self.question_target[1]]: self.reward -= 1.0 elif action['objectId'] not in self.opened_receptacles: if self.question_type_ind == 2 and action[ 'objectId'].split('|')[0] == constants.OBJECTS[ self.question_target[1]]: self.reward += 5.0 else: self.reward += 0.1 self.opened_receptacles.add(action['objectId']) elif action[ 'action'] == 'CloseObject' and self.question_type_ind != 2: if action['objectId'] not in self.closed_receptacles: self.reward += 0.1 self.closed_receptacles.add(action['objectId']) # Update seen objects related to question objs = game_util.get_objects_of_type( constants.OBJECTS[self.object_target], self.event.metadata) objs = [ obj for obj in objs if (obj['objectId'] in self.event.instance_detections2D and game_util.check_object_size( self.event.instance_detections2D[obj['objectId']])) ] for obj in objs: self.seen_obj1.add(obj['objectId']) if self.question_type_ind in {2, 3}: objs = game_util.get_objects_of_type( constants.OBJECTS[self.question_target[1]], self.event.metadata) objs = [ obj for obj in objs if (obj['objectId'] in self.event.instance_detections2D and game_util.check_object_size( self.event.instance_detections2D[obj['objectId']])) ] for obj in objs: self.seen_obj2.add(obj['objectId']) if not self.can_end: if self.question_type_ind == 0: self.can_end = len(self.seen_obj1) > 0 elif self.question_type_ind == 1: self.can_end = len(self.seen_obj1) == self.answer elif self.question_type_ind == 2: objs = game_util.get_objects_of_type( constants.OBJECTS[self.question_target[1]], self.event.metadata) if not self.answer: if objs[0]['openable']: if all([ obj['objectId'] in self.opened_receptacles for obj in objs ]): self.can_end = True else: if all([ obj['objectId'] in self.seen_obj2 for obj in objs ]): self.can_end = True else: objs = [ obj for obj in objs if (obj['objectId'] in self.event.instance_detections2D and game_util.check_object_size( self.event.instance_detections2D[ obj['objectId']])) ] for obj in objs: for contained_obj in obj['pivotSimObjs']: if contained_obj['objectId'] in self.seen_obj1: self.can_end = True else: self.reward -= 0.05
def reset(self, seed=None, test_ind=None): self.board = None self.seen_object = False self.terminal = False self.opened_receptacles = set() self.closed_receptacles = set() self.seen_obj1 = set() self.seen_obj2 = set() self.visited_locations = set() self.can_end = False if seed is not None: self.local_random.seed(seed) # Do equal number of each question type in train. question_type_ind = self.local_random.sample( constants.USED_QUESTION_TYPES, 1)[0] # get random row if test_ind is not None: question_row, question_type_ind = test_ind question_type = self.question_types[question_type_ind] question_data = self.test_datasets[question_type_ind][ question_row, :] test_ind = (question_row, question_type_ind) else: question_type = self.question_types[question_type_ind] question_row = self.local_random.randint( 0, len(self.datasets[question_type_ind]) - 1) question_data = self.datasets[question_type_ind][question_row, :] container_ind = None if question_type_ind == 0 or question_type_ind == 1: scene_num, scene_seed, object_ind, answer = question_data self.question_target = object_ind if question_type_ind == 0: answer = bool(answer) elif question_type_ind == 2: scene_num, scene_seed, object_ind, container_ind, answer = question_data answer = bool(answer) self.question_target = (object_ind, container_ind) else: raise Exception('No question type found for type %d' % question_type_ind) self.scene_seed = scene_seed self.scene_num = scene_num self.object_target = object_ind self.parent_target = container_ind self.container_target = np.zeros(constants.NUM_CLASSES) self.direction_target = np.zeros(4) if container_ind is not None: self.container_target[container_ind] = 1 self.question_type_ind = question_type_ind self.scene_name = 'FloorPlan%d' % scene_num grid_file = 'layouts/%s-layout.npy' % self.scene_name self.graph = graph_obj.Graph(grid_file, use_gt=False) self.xray_graph = graph_obj.Graph(grid_file, use_gt=True) self.bounds = [ self.graph.xMin, self.graph.yMin, self.graph.xMax - self.graph.xMin + 1, self.graph.yMax - self.graph.yMin + 1 ] max_num_repeats = 1 remove_prob = 0.5 if question_type == 'existence': max_num_repeats = 10 remove_prob = 0.25 elif question_type == 'counting': max_num_repeats = constants.MAX_COUNTING_ANSWER + 1 remove_prob = 0.5 elif question_type == 'contains': max_num_repeats = 10 remove_prob = 0.25 self.event = game_util.reset(self.env, self.scene_name) self.agent_height = self.event.metadata['agent']['position']['y'] self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET self.event = self.env.random_initialize( self.scene_seed, max_num_repeats=max_num_repeats, remove_prob=remove_prob) print('Type:', question_type, 'Row: ', question_row, 'Scene', self.scene_name, 'seed', scene_seed) print( 'Question:', game_util.get_question_str(question_type_ind, object_ind, container_ind)) if self.question_type_ind == 2: print('Answer:', constants.OBJECTS[object_ind], 'in', constants.OBJECTS[container_ind], 'is', answer) else: print('Answer:', constants.OBJECTS[object_ind], 'is', answer) self.answer = answer # Verify answer if self.question_type_ind == 0: objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind], self.event.metadata) computed_answer = len(objs) > 0 requires_interaction = True for obj in objs: parent = obj['parentReceptacle'].split('|')[0] if parent not in {'Fridge', 'Cabinet', 'Microwave'}: requires_interaction = False break elif self.question_type_ind == 1: objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind], self.event.metadata) computed_answer = len(objs) requires_interaction = True elif self.question_type_ind == 2: objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind], self.event.metadata) if len(objs) == 0: computed_answer = False requires_interaction = constants.OBJECTS[ self.question_target[1]] in { 'Fridge', 'Cabinet', 'Microwave' } else: obj = objs[0] computed_answer = False for obj in objs: requires_interaction = True parent = obj['parentReceptacle'].split('|')[0] if parent in constants.OBJECT_CLASS_TO_ID: parent_ind = constants.OBJECT_CLASS_TO_ID[parent] computed_answer = parent_ind == self.question_target[1] if computed_answer: if parent not in { 'Fridge', 'Cabinet', 'Microwave' }: requires_interaction = False break else: computed_answer = False self.requires_interaction = requires_interaction try: assert self.answer == computed_answer, 'Answer does not match scene metadata' except AssertionError: print('Type:', question_type, 'Row: ', question_row, 'Scene', self.scene_name, 'seed', scene_seed) print('Answer', computed_answer, 'does not match expected value', self.answer, ', did randomization process change?') pdb.set_trace() self.answer = computed_answer if constants.NUM_CLASSES > 1: self.hidden_items = set() objects = self.event.metadata['objects'] for obj in objects: if obj['receptacle'] and obj['openable'] and not obj['isopen']: for inside_obj in obj['receptacleObjectIds']: self.hidden_items.add(inside_obj) objects = self.event.metadata['objects'] for obj in objects: if obj['objectType'] not in constants.OBJECT_CLASS_TO_ID: continue obj_bounds = game_util.get_object_bounds(obj, self.bounds) self.xray_graph.memory[ obj_bounds[1]:obj_bounds[3], obj_bounds[0]:obj_bounds[2], constants.OBJECT_CLASS_TO_ID[obj['objectType']] + 1] = 1 start_point = self.local_random.randint(0, self.graph.points.shape[0] - 1) start_point = self.graph.points[start_point, :].copy() self.start_point = (start_point[0], start_point[1], self.local_random.randint(0, 3)) action = { 'action': 'TeleportFull', 'x': self.start_point[0] * constants.AGENT_STEP_SIZE, 'y': self.agent_height, 'z': self.start_point[1] * constants.AGENT_STEP_SIZE, 'rotateOnTeleport': True, 'rotation': self.start_point[2] * 90, 'horizon': 30, } self.event = self.env.step(action) self.process_frame() self.reward = 0 self.end_point = []
def setup_problem(self, seed=None, info=None, scene=None, objs=None): ''' setup goal with sampled objects or with the objects specified note: must be used with `initialize_random_scene` ''' self.terminal = False print( 'setup random goal ----------------------------------------------') print('seed', seed) print('info', info) print( '--------------------------------------------------------------------\n\n' ) super(TaskGameState, self).setup_problem(seed) info, max_num_repeats, remove_prob = self.get_random_task_vals( scene=scene) dataset_type, task_row = info print('Type:', dataset_type, 'Row: ', task_row, 'Scene', self.scene_name, 'seed', self.scene_seed) perform_sanity_check = True # pickupable, receptacle, toggleable candidates pickupable_objects = [ o for o in self.env.last_event.metadata['objects'] if (o['pickupable'] and o['objectType'] in constants.OBJECTS) ] receptacle_objects = [ o for o in self.env.last_event.metadata['objects'] if (o['receptacle'] and o['objectType'] in constants.RECEPTACLES and o['objectType'] not in constants.MOVABLE_RECEPTACLES_SET) ] toggle_objects = [ o for o in self.env.last_event.metadata['objects'] if o['toggleable'] and not o['isToggled'] # must be one we can turn on. and o['objectType'] in constants.VAL_ACTION_OBJECTS["Toggleable"] ] if len(pickupable_objects) == 0: print("Task Failed - %s" % constants.pddl_goal_type) raise Exception("No pickupable objects in the scene") # filter based on crit obj_crit, _ = self.get_filter_crit(constants.pddl_goal_type) pickupable_objects = list(filter(obj_crit, pickupable_objects)) if len(pickupable_objects) == 0: print("Task Failed - %s" % constants.pddl_goal_type) raise Exception("No pickupable objects related to the goal '%s'" % constants.pddl_goal_type) # choose a pickupable object if constants.FORCED_SAMPLING or objs is None: self.rand_chosen_object = random.choice(pickupable_objects) else: chosen_class_available = [ obj for obj in pickupable_objects if obj['objectType'] == constants.OBJ_PARENTS[objs['pickup']] ] if len(chosen_class_available) == 0: print("Task Failed - %s" % constants.pddl_goal_type) raise Exception( "Couldn't find a valid parent '%s' for pickup object with class '%s'" % (constants.OBJ_PARENTS[objs['pickup']], objs['pickup'])) self.rand_chosen_object = random.choice(chosen_class_available) self.rand_chosen_object_class = self.rand_chosen_object['objectType'] self.object_target = constants.OBJECTS.index( self.rand_chosen_object_class) # for now, any obj differing from its parent is obtained via slicing, but that may change in the future. if constants.OBJ_PARENTS[objs['pickup']] != objs['pickup']: requires_slicing = True else: requires_slicing = False # choose a movable receptacle if "movable_recep" in constants.pddl_goal_type: val_movable_receps = [ o for o in self.env.last_event.metadata['objects'] if (o['objectType'] in constants.MOVABLE_RECEPTACLES_SET) and ( self.rand_chosen_object_class in constants.VAL_RECEPTACLE_OBJECTS[o['objectType']]) ] if len(val_movable_receps) == 0: print("Task Failed - %s" % constants.pddl_goal_type) raise Exception( "Couldn't find a valid moveable receptacle for the chosen object class" ) if objs is not None: val_movable_receps = [ o for o in val_movable_receps if o['objectType'] == objs['mrecep'] ] self.rand_chosen_val_moveable_recep_class = random.choice( val_movable_receps)['objectType'] self.mrecep_target = constants.OBJECTS.index( self.rand_chosen_val_moveable_recep_class) else: self.mrecep_target = None # if slicing is required, make sure a knife is available in the scene if requires_slicing: knife_objs = [ o for o in self.env.last_event.metadata['objects'] if ("Knife" in o['objectType']) ] if len(knife_objs) == 0: print("Task Failed - %s" % constants.pddl_goal_type) raise Exception("Couldn't find knife in the scene to cut with") if constants.pddl_goal_type == "look_at_obj_in_light": # choose a toggleable object self.parent_target = None if constants.FORCED_SAMPLING or objs is None: rand_chosen_toggle_object = random.choice(toggle_objects) else: toggle_class_available = [ obj for obj in toggle_objects if obj['objectType'] == objs['toggle'] ] if len(toggle_class_available) == 0: print("Task Failed - %s" % constants.pddl_goal_type) raise Exception( "Couldn't find a valid toggle object of class '%s'" % objs['toggle']) rand_chosen_toggle_object = random.choice( toggle_class_available) rand_chosen_toggle_class = rand_chosen_toggle_object['objectType'] self.toggle_target = constants.OBJECTS.index( rand_chosen_toggle_class) else: self.toggle_target = None # find all valid receptacles in which rand_chosen object or rand chosen moveable receptacle can be placed val_receptacle_objects_orig = [ r for r in receptacle_objects if (self.rand_chosen_val_moveable_recep_class if self. mrecep_target != None else self.rand_chosen_object_class ) in constants.VAL_RECEPTACLE_OBJECTS[r['objectType']] ] _, recep_crit = self.get_filter_crit(constants.pddl_goal_type) val_receptacle_objects = list( filter(recep_crit, val_receptacle_objects_orig)) if len(val_receptacle_objects) == 0: print("Task Failed - %s" % constants.pddl_goal_type) raise Exception( "Couldn't find a valid receptacle object according to constraints specified" ) # choose a receptacle object if constants.FORCED_SAMPLING or objs is None: rand_chosen_receptacle_object = random.choice( val_receptacle_objects) else: receptacle_class_available = [ obj for obj in val_receptacle_objects if obj['objectType'] == objs['receptacle'] ] if len(receptacle_class_available) == 0: print("Task Failed - %s" % constants.pddl_goal_type) raise Exception( "Couldn't find a valid receptacle object of class '%s'" % objs['receptacle']) rand_chosen_receptacle_object = random.choice( receptacle_class_available) rand_chosen_receptacle_object_class = rand_chosen_receptacle_object[ 'objectType'] self.parent_target = constants.OBJECTS.index( rand_chosen_receptacle_object_class) # pddl_param dict constants.data_dict['pddl_params'][ 'object_target'] = constants.OBJECTS[ self.object_target] if self.object_target is not None else "" constants.data_dict['pddl_params']['object_sliced'] = requires_slicing constants.data_dict['pddl_params'][ 'parent_target'] = constants.OBJECTS[ self.parent_target] if self.parent_target is not None else "" constants.data_dict['pddl_params'][ 'toggle_target'] = constants.OBJECTS[ self.toggle_target] if self.toggle_target is not None else "" constants.data_dict['pddl_params'][ 'mrecep_target'] = constants.OBJECTS[ self.mrecep_target] if self.mrecep_target is not None else "" self.task_target = (self.object_target, self.parent_target, self.toggle_target, self.mrecep_target) if self.parent_target is None: validity_check = True else: objs = game_util.get_objects_of_type( constants.OBJECTS[self.object_target], self.env.last_event.metadata) available_receptacles = game_util.get_objects_of_type( constants.OBJECTS[self.parent_target], self.env.last_event.metadata) validity_check = (len(objs) > 0 and len([obj for obj in available_receptacles])) if perform_sanity_check: try: assert validity_check, 'Task does not seem valid according to scene metadata' except AssertionError: raise Exception( str(('Row: ', task_row, 'Scene', self.scene_name, 'seed', self.scene_seed))) templated_task_desc = self.get_task_str() print('problem id', self.problem_id) print('Task:', templated_task_desc) constants.data_dict['template']['task_desc'] = templated_task_desc
def does_have_atleast_n_duplicates(x, n): num_instances = len( game_util.get_objects_of_type(x['objectType'], self.env.last_event.metadata)) return num_instances >= n