def set_state(self, state: AgentState, reset_sensors: bool = True): r"""Sets the agents state :param state: The state to set the agent to :param reset_sensors: Whether or not to reset the sensors to their default intrinsic/extrinsic parameters before setting their extrinsic state """ habitat_sim.errors.assert_obj_valid(self.body) if isinstance(state.rotation, list): state.rotation = quat_from_coeffs(state.rotation) self.body.object.reset_transformation() self.body.object.translate(state.position) self.body.object.rotation = quat_to_magnum(state.rotation) if reset_sensors: for _, v in self._sensors.items(): v.set_transformation_from_spec() for k, v in state.sensor_states.items(): assert k in self._sensors if isinstance(v.rotation, list): v.rotation = quat_from_coeffs(v.rotation) s = self._sensors[k] s.node.reset_transformation() s.node.translate( quat_rotate_vector(state.rotation.inverse(), v.position - state.position)) s.node.rotation = quat_to_magnum(state.rotation.inverse() * v.rotation)
def set_state( self, state: AgentState, reset_sensors: bool = True, infer_sensor_states: bool = True, is_initial: bool = False, ) -> None: r"""Sets the agents state :param state: The state to set the agent to :param reset_sensors: Whether or not to reset the sensors to their default intrinsic/extrinsic parameters before setting their extrinsic state. :param infer_sensor_states: Whether or not to infer the location of sensors based on the new location of the agent base state. :param is_initial: Whether this state is the initial state of the agent in the scene. Used for resetting the agent at a later time Setting ``reset_sensors`` to :py:`False` allows the agent base state to be moved and the new sensor locations inferred without changing the configuration of the sensors with respect to the base state of the agent. Setting ``infer_sensor_states`` to :py:`False` is useful if you'd like to directly control the state of a sensor instead of moving the agent. """ attr.validate(state) habitat_sim.errors.assert_obj_valid(self.body) if isinstance(state.rotation, (list, np.ndarray)): state.rotation = quat_from_coeffs(state.rotation) self.body.object.reset_transformation() self.body.object.translate(state.position) self.body.object.rotation = quat_to_magnum(state.rotation) if reset_sensors: for _, v in self._sensors.items(): v.set_transformation_from_spec() if not infer_sensor_states: for k, v in state.sensor_states.items(): assert k in self._sensors if isinstance(v.rotation, list): v.rotation = quat_from_coeffs(v.rotation) s = self._sensors[k] s.node.reset_transformation() s.node.translate( quat_rotate_vector(state.rotation.inverse(), v.position - state.position)) s.node.rotation = quat_to_magnum(state.rotation.inverse() * v.rotation) if is_initial: self.initial_state = state
def reconfigure(self, config: Config) -> None: self.config = config is_same_sound = config.AGENT_0.SOUND_ID == self._current_sound if not is_same_sound: self._current_sound = self.config.AGENT_0.SOUND_ID is_same_scene = config.SCENE == self._current_scene if not is_same_scene: self._current_scene = config.SCENE logging.debug('Current scene: {} and sound: {}'.format(self.current_scene_name, self._current_sound)) if not self.config.USE_RENDERED_OBSERVATIONS: self._sim.close() del self._sim self.sim_config = self.create_sim_config(self._sensor_suite) self._sim = habitat_sim.Simulator(self.sim_config) self._update_agents_state() self._frame_cache = dict() else: with open(self.current_scene_observation_file, 'rb') as fo: self._frame_cache = pickle.load(fo) logging.debug('Loaded scene {}'.format(self.current_scene_name)) self.points, self.graph = load_metadata(self.metadata_dir) for node in self.graph.nodes(): self._position_to_index_mapping[self.position_encoding(self.graph.nodes()[node]['point'])] = node if not is_same_scene or not is_same_sound: self._audiogoal_cache = dict() self._spectrogram_cache = dict() self._episode_step_count = 0 # set agent positions self._receiver_position_index = self._position_to_index(self.config.AGENT_0.START_POSITION) self._source_position_index = self._position_to_index(self.config.AGENT_0.GOAL_POSITION) # the agent rotates about +Y starting from -Z counterclockwise, # so rotation angle 90 means the agent rotate about +Y 90 degrees self._rotation_angle = int(np.around(np.rad2deg(quat_to_angle_axis(quat_from_coeffs( self.config.AGENT_0.START_ROTATION))[0]))) % 360 if not self.config.USE_RENDERED_OBSERVATIONS: self.set_agent_state(list(self.graph.nodes[self._receiver_position_index]['point']), self.config.AGENT_0.START_ROTATION) else: self._sim.set_agent_state(list(self.graph.nodes[self._receiver_position_index]['point']), quat_from_coeffs(self.config.AGENT_0.START_ROTATION)) logging.debug("Initial source, agent at: {}, {}, orientation: {}". format(self._source_position_index, self._receiver_position_index, self.get_orientation()))
def reconfigure(self, config) -> None: dataset = config.SCENE.split('/')[2] scene_name = config.SCENE.split('/')[3] is_same_scene = config.SCENE == self._current_scene self.config = config self.sim_config = self.create_sim_config(self._sensor_suite) if not is_same_scene: self._current_scene = config.SCENE self._sim.close() del self._sim # HabitatSim is a wrapper class of habitat_sim, which is the backend renderer self._sim = habitat_sim.Simulator(self.sim_config) logging.info('Loaded scene {}'.format(scene_name)) if not is_same_scene or self.points is None or self.graph is None: # can happen during initialization or reconfiguration metadata_dir = os.path.join('data/metadata', dataset, scene_name) self.points, self.graph = load_metadata(metadata_dir) # after env calls reconfigure to update the config with current episode, # simulation needs to update the agent position, rotation in accordance with the new config file self._update_agents_state() # set agent positions self._receiver_position_index = self._position_to_index(self.config.AGENT_0.START_POSITION) self._source_position_index = self._position_to_index(self.config.AGENT_0.GOAL_POSITION) self._rotation = int(np.around(np.rad2deg(quat_to_angle_axis(quat_from_coeffs( self.config.AGENT_0.START_ROTATION))[0]))) % 360 self.set_agent_state(list(self.graph.nodes[self._receiver_position_index]['point']), self.config.AGENT_0.START_ROTATION) logging.debug("Initial source, agent at: {}, {}, orientation: {}". format(self._source_position_index, self._receiver_position_index, self._rotation))
def _render_and_load_gt(sim, scene, sensor_type, gpu2gpu): gt_data_pose_file = osp.abspath( osp.join( osp.dirname(__file__), "gt_data", "{}-state.json".format(osp.basename(osp.splitext(scene)[0])), )) with open(gt_data_pose_file, "r") as f: render_state = json.load(f) state = habitat_sim.AgentState() state.position = render_state["pos"] state.rotation = quat_from_coeffs(render_state["rot"]) sim.initialize_agent(0, state) obs = sim.step("move_forward") assert sensor_type in obs, f"{sensor_type} not in obs" gt_obs_file = osp.abspath( osp.join( osp.dirname(__file__), "gt_data", "{}-{}.npy".format(osp.basename(osp.splitext(scene)[0]), sensor_type), )) gt = np.load(gt_obs_file) if gpu2gpu: torch = pytest.importorskip("torch") for k, v in obs.items(): if torch.is_tensor(v): obs[k] = v.cpu().numpy() return obs, gt
def test_sensors(scene, has_sem, sensor_type, gpu2gpu, sim, make_cfg_settings): if not osp.exists(scene): pytest.skip("Skipping {}".format(scene)) if not habitat_sim.cuda_enabled and gpu2gpu: pytest.skip("Skipping GPU->GPU test") make_cfg_settings = {k: v for k, v in make_cfg_settings.items()} make_cfg_settings["semantic_sensor"] = has_sem make_cfg_settings["scene"] = scene cfg = make_cfg(make_cfg_settings) for sensor_spec in cfg.agents[0].sensor_specifications: sensor_spec.gpu2gpu_transfer = gpu2gpu sim.reconfigure(cfg) with open( osp.abspath( osp.join( osp.dirname(__file__), "gt_data", "{}-state.json".format(osp.basename(osp.splitext(scene)[0])), ) ), "r", ) as f: render_state = json.load(f) state = habitat_sim.AgentState() state.position = render_state["pos"] state.rotation = quat_from_coeffs(render_state["rot"]) sim.initialize_agent(0, state) obs = sim.step("move_forward") assert sensor_type in obs, f"{sensor_type} not in obs" gt = np.load( osp.abspath( osp.join( osp.dirname(__file__), "gt_data", "{}-{}.npy".format(osp.basename(osp.splitext(scene)[0]), sensor_type), ) ) ) if gpu2gpu: import torch for k, v in obs.items(): if torch.is_tensor(v): obs[k] = v.cpu().numpy() # Different GPUs and different driver version will produce slightly different images assert np.linalg.norm( obs[sensor_type].astype(np.float) - gt.astype(np.float) ) < 1.5e-2 * np.linalg.norm(gt.astype(np.float)), f"Incorrect {sensor_type} output"
# @markdown --- # @markdown Configure Parameters: obj_attr_mgr = sim.get_object_template_manager() remove_all_objects(sim) seed = 2 # @param{type:"integer"} random.seed(seed) sim.seed(seed) np.random.seed(seed) # setup agent state manually to face the bar agent_state = sim.agents[0].state agent_state.position = np.array([-1.97496, 0.072447, -2.0894]) agent_state.rotation = ut.quat_from_coeffs([0, -1, 0, 0]) sim.agents[0].set_state(agent_state) # load the target objects cheezit_handle = obj_attr_mgr.get_template_handles("cheezit")[0] # create range from center and half-extent target_zone = mn.Range3D.from_center( mn.Vector3(-2.07496, 1.07245, -0.2894), mn.Vector3(0.5, 0.05, 0.1) ) num_targets = 9 # @param{type:"integer"} for _target in range(num_targets): obj_id = sim.add_object_by_handle(cheezit_handle) # rotate boxes off of their sides rotate = mn.Quaternion.rotation(mn.Rad(-mn.math.pi_half), mn.Vector3(1.0, 0, 0)) sim.set_rotation(rotate, obj_id) # sample state from the target zone
def reconfigure(self, config: Config) -> None: self.config = config if hasattr(self.config.AGENT_0, 'OFFSET'): self._offset = int(self.config.AGENT_0.OFFSET) else: self._offset = 0 if self.config.AUDIO.EVERLASTING: self._duration = 500 else: assert hasattr(self.config.AGENT_0, 'DURATION') self._duration = int(self.config.AGENT_0.DURATION) self._audio_index = 0 is_same_sound = config.AGENT_0.SOUND_ID == self._current_sound if not is_same_sound: self._current_sound = self.config.AGENT_0.SOUND_ID self._load_single_source_sound() logging.debug("Switch to sound {} with duration {} seconds".format(self._current_sound, self._duration)) is_same_scene = config.SCENE == self._current_scene if not is_same_scene: self._current_scene = config.SCENE logging.debug('Current scene: {} and sound: {}'.format(self.current_scene_name, self._current_sound)) if self.config.USE_RENDERED_OBSERVATIONS: with open(self.current_scene_observation_file, 'rb') as fo: self._frame_cache = pickle.load(fo) else: self._sim.close() del self._sim self.sim_config = self.create_sim_config(self._sensor_suite) self._sim = habitat_sim.Simulator(self.sim_config) self._update_agents_state() self._frame_cache = dict() logging.debug('Loaded scene {}'.format(self.current_scene_name)) self.points, self.graph = load_metadata(self.metadata_dir) for node in self.graph.nodes(): self._position_to_index_mapping[self.position_encoding(self.graph.nodes()[node]['point'])] = node self._instance2label_mapping = None if not is_same_scene or not is_same_sound: self._audiogoal_cache = dict() self._spectrogram_cache = dict() self._episode_step_count = 0 # set agent positions self._receiver_position_index = self._position_to_index(self.config.AGENT_0.START_POSITION) self._source_position_index = self._position_to_index(self.config.AGENT_0.GOAL_POSITION) # the agent rotates about +Y starting from -Z counterclockwise, # so rotation angle 90 means the agent rotate about +Y 90 degrees self._rotation_angle = int(np.around(np.rad2deg(quat_to_angle_axis(quat_from_coeffs( self.config.AGENT_0.START_ROTATION))[0]))) % 360 if self.config.USE_RENDERED_OBSERVATIONS: self._sim.set_agent_state(list(self.graph.nodes[self._receiver_position_index]['point']), quat_from_coeffs(self.config.AGENT_0.START_ROTATION)) else: self.set_agent_state(list(self.graph.nodes[self._receiver_position_index]['point']), self.config.AGENT_0.START_ROTATION) if self.config.AUDIO.HAS_DISTRACTOR_SOUND: self._distractor_position_index = self.config.AGENT_0.DISTRACTOR_POSITION_INDEX self._current_distractor_sound = self.config.AGENT_0.DISTRACTOR_SOUND_ID self._load_single_distractor_sound() if self._use_oracle_planner: self._oracle_actions = self.compute_oracle_actions() logging.debug("Initial source, agent at: {}, {}, orientation: {}". format(self._source_position_index, self._receiver_position_index, self.get_orientation()))