def main():
metadata_folder = os.path.join('data/metadata/mp3d')
scenes = os.listdir(metadata_folder)
for scene in scenes:
navmesh_file = "data/scene_datasets/mp3d/{}/{}.navmesh".format(
scene, scene)
scene_metadata_folder = os.path.join(metadata_folder, scene)
graph_file = os.path.join(scene_metadata_folder, 'graph.pkl')
visualization_dir = 'data/visualizations/mp3d'
os.makedirs(scene_metadata_folder, exist_ok=True)
os.makedirs(visualization_dir, exist_ok=True)
pathfinder = hsim.PathFinder()
pathfinder.load_nav_mesh(navmesh_file)
points, _ = load_metadata(scene_metadata_folder)
graph = generate_graph(points, pathfinder)
visualize(points,
graph,
scene,
save_figure=True,
plot_indices=True,
output_dir=visualization_dir)
adjusted = adjust_graph(graph, points, scene)
if adjusted:
visualize(points,
graph,
scene + '_fix',
save_figure=True,
plot_indices=True,
output_dir=visualization_dir)
with open(graph_file, 'wb') as fo:
pickle.dump(graph, fo)
def reconfigure(self, config) -> None:
dataset = config.SCENE.split('/')[2]
scene_name = config.SCENE.split('/')[3]
is_same_scene = config.SCENE == self._current_scene
self.config = config
self.sim_config = self.create_sim_config(self._sensor_suite)
if not is_same_scene:
self._current_scene = config.SCENE
self._sim.close()
del self._sim
# HabitatSim is a wrapper class of habitat_sim, which is the backend renderer
self._sim = habitat_sim.Simulator(self.sim_config)
logging.info('Loaded scene {}'.format(scene_name))
if not is_same_scene or self.points is None or self.graph is None:
# can happen during initialization or reconfiguration
metadata_dir = os.path.join('data/metadata', dataset, scene_name)
self.points, self.graph = load_metadata(metadata_dir)
# after env calls reconfigure to update the config with current episode,
# simulation needs to update the agent position, rotation in accordance with the new config file
self._update_agents_state()
# set agent positions
self._receiver_position_index = self._position_to_index(self.config.AGENT_0.START_POSITION)
self._source_position_index = self._position_to_index(self.config.AGENT_0.GOAL_POSITION)
self._rotation = int(np.around(np.rad2deg(quat_to_angle_axis(quat_from_coeffs(
self.config.AGENT_0.START_ROTATION))[0]))) % 360
self.set_agent_state(list(self.graph.nodes[self._receiver_position_index]['point']),
self.config.AGENT_0.START_ROTATION)
logging.debug("Initial source, agent at: {}, {}, orientation: {}".
format(self._source_position_index, self._receiver_position_index,
self._rotation))
def __init__(self, config: Config) -> None:
super().__init__(config)
self._source_position_index = None
self._receiver_position_index = None
self._rotation_angle = None
self._current_sound = None
self._source_sound_dict = dict()
self._sampling_rate = None
self._node2index = None
self._frame_cache = dict()
self._audiogoal_cache = dict()
self._spectrogram_cache = dict()
self._scene_observations = None
self._episode_step_count = None
self._is_episode_active = None
self._position_to_index_mapping = dict()
self._previous_step_collided = False
self.points, self.graph = load_metadata(self.metadata_dir)
for node in self.graph.nodes():
self._position_to_index_mapping[self.position_encoding(
self.graph.nodes()[node]['point'])] = node
self._load_sound_sources()
logging.info('Current scene: {} and sound: {}'.format(
self.current_scene_name, self._current_sound))
if self.config.USE_RENDERED_OBSERVATIONS:
self._sim.close()
del self._sim
self._sim = DummySimulator()
with open(self.current_scene_observation_file, 'rb') as fo:
self._frame_cache = pickle.load(fo)
def __init__(self, config: Config) -> None:
self.config = config
agent_config = self._get_agent_config()
sim_sensors = []
for sensor_name in agent_config.SENSORS:
sensor_cfg = getattr(self.config, sensor_name)
sensor_type = registry.get_sensor(sensor_cfg.TYPE)
assert sensor_type is not None, "invalid sensor type {}".format(
sensor_cfg.TYPE
)
sim_sensors.append(sensor_type(sensor_cfg))
self._sensor_suite = SensorSuite(sim_sensors)
self.sim_config = self.create_sim_config(self._sensor_suite)
self._current_scene = self.sim_config.sim_cfg.scene_id
self._action_space = spaces.Discrete(
len(self.sim_config.agents[0].action_space)
)
self._prev_sim_obs = None
self._source_position_index = None
self._receiver_position_index = None
self._rotation_angle = None
self._current_sound = None
self._offset = None
self._duration = None
self._audio_index = None
self._audio_length = None
self._source_sound_dict = dict()
self._sampling_rate = None
self._node2index = None
self._frame_cache = dict()
self._audiogoal_cache = dict()
self._spectrogram_cache = dict()
self._egomap_cache = defaultdict(dict)
self._scene_observations = None
self._episode_step_count = None
self._is_episode_active = None
self._position_to_index_mapping = dict()
self._previous_step_collided = False
self._instance2label_mapping = None
self._house_readers = dict()
self._use_oracle_planner = True
self._oracle_actions = list()
self.points, self.graph = load_metadata(self.metadata_dir)
for node in self.graph.nodes():
self._position_to_index_mapping[self.position_encoding(self.graph.nodes()[node]['point'])] = node
if self.config.AUDIO.HAS_DISTRACTOR_SOUND:
self._distractor_position_index = None
self._current_distractor_sound = None
if self.config.USE_RENDERED_OBSERVATIONS:
self._sim = DummySimulator()
with open(self.current_scene_observation_file, 'rb') as fo:
self._frame_cache = pickle.load(fo)
else:
self._sim = habitat_sim.Simulator(config=self.sim_config)
def __init__(self, config: Config) -> None:
self.config = config
agent_config = self.get_agent_config()
sim_sensors = []
for sensor_name in agent_config.SENSORS:
sensor_cfg = getattr(self.config, sensor_name)
sensor_type = registry.get_sensor(sensor_cfg.TYPE)
assert sensor_type is not None, "invalid sensor type {}".format(
sensor_cfg.TYPE
)
sim_sensors.append(sensor_type(sensor_cfg))
self._sensor_suite = SensorSuite(sim_sensors)
self.sim_config = self.create_sim_config(self._sensor_suite)
self._current_scene = self.sim_config.sim_cfg.scene.id
self._sim = habitat_sim.Simulator(self.sim_config)
self._action_space = spaces.Discrete(
len(self.sim_config.agents[0].action_space)
)
self._prev_sim_obs = None
self._source_position_index = None
self._receiver_position_index = None
self._rotation_angle = None
self._current_sound = None
self._source_sound_dict = dict()
self._sampling_rate = None
self._node2index = None
self._frame_cache = dict()
self._audiogoal_cache = dict()
self._spectrogram_cache = dict()
self._egomap_cache = defaultdict(dict)
self._scene_observations = None
self._episode_step_count = None
self._is_episode_active = None
self._position_to_index_mapping = dict()
self._previous_step_collided = False
self.points, self.graph = load_metadata(self.metadata_dir)
for node in self.graph.nodes():
self._position_to_index_mapping[self.position_encoding(self.graph.nodes()[node]['point'])] = node
self._load_source_sounds()
logging.info('Current scene: {} and sound: {}'.format(self.current_scene_name, self._current_sound))
if self.config.USE_RENDERED_OBSERVATIONS:
self._sim.close()
del self._sim
self._sim = DummySimulator()
with open(self.current_scene_observation_file, 'rb') as fo:
self._frame_cache = pickle.load(fo)
def reconfigure(self, config: Config) -> None:
self.config = config
is_same_sound = config.AGENT_0.SOUND_ID == self._current_sound
if not is_same_sound:
self._current_sound = self.config.AGENT_0.SOUND_ID
is_same_scene = config.SCENE == self._current_scene
if not is_same_scene:
self._current_scene = config.SCENE
logging.debug('Current scene: {} and sound: {}'.format(self.current_scene_name, self._current_sound))
if not self.config.USE_RENDERED_OBSERVATIONS:
self._sim.close()
del self._sim
self.sim_config = self.create_sim_config(self._sensor_suite)
self._sim = habitat_sim.Simulator(self.sim_config)
self._update_agents_state()
self._frame_cache = dict()
else:
with open(self.current_scene_observation_file, 'rb') as fo:
self._frame_cache = pickle.load(fo)
logging.debug('Loaded scene {}'.format(self.current_scene_name))
self.points, self.graph = load_metadata(self.metadata_dir)
for node in self.graph.nodes():
self._position_to_index_mapping[self.position_encoding(self.graph.nodes()[node]['point'])] = node
if not is_same_scene or not is_same_sound:
self._audiogoal_cache = dict()
self._spectrogram_cache = dict()
self._episode_step_count = 0
# set agent positions
self._receiver_position_index = self._position_to_index(self.config.AGENT_0.START_POSITION)
self._source_position_index = self._position_to_index(self.config.AGENT_0.GOAL_POSITION)
# the agent rotates about +Y starting from -Z counterclockwise,
# so rotation angle 90 means the agent rotate about +Y 90 degrees
self._rotation_angle = int(np.around(np.rad2deg(quat_to_angle_axis(quat_from_coeffs(
self.config.AGENT_0.START_ROTATION))[0]))) % 360
if not self.config.USE_RENDERED_OBSERVATIONS:
self.set_agent_state(list(self.graph.nodes[self._receiver_position_index]['point']),
self.config.AGENT_0.START_ROTATION)
else:
self._sim.set_agent_state(list(self.graph.nodes[self._receiver_position_index]['point']),
quat_from_coeffs(self.config.AGENT_0.START_ROTATION))
logging.debug("Initial source, agent at: {}, {}, orientation: {}".
format(self._source_position_index, self._receiver_position_index, self.get_orientation()))
def main(dataset):
parser = argparse.ArgumentParser()
parser.add_argument(
"--config-path",
type=str,
default='baselines/config/{}/train_telephone/pointgoal_rgb.yaml'.format(dataset)
)
parser.add_argument(
"opts",
default=None,
nargs=argparse.REMAINDER,
help="Modify config options from command line",
)
args = parser.parse_args()
config = get_config(args.config_path, opts=args.opts)
config.defrost()
config.TASK_CONFIG.SIMULATOR.AGENT_0.SENSORS = ["RGB_SENSOR", "DEPTH_SENSOR"]
config.freeze()
simulator = None
scene_obs = defaultdict(dict)
num_obs = 0
scene_obs_dir = 'data/scene_observations/' + dataset
os.makedirs(scene_obs_dir, exist_ok=True)
metadata_dir = 'data/metadata/' + dataset
for scene in os.listdir(metadata_dir):
scene_obs = dict()
scene_metadata_dir = os.path.join(metadata_dir, scene)
points, graph = load_metadata(scene_metadata_dir)
if dataset == 'replica':
scene_mesh_dir = os.path.join('data/scene_datasets', dataset, scene, 'habitat/mesh_semantic.ply')
else:
scene_mesh_dir = os.path.join('data/scene_datasets', dataset, scene, scene + '.glb')
for node in graph.nodes():
agent_position = graph.nodes()[node]['point']
for angle in [0, 90, 180, 270]:
agent_rotation = quat_to_coeffs(quat_from_angle_axis(np.deg2rad(angle), np.array([0, 1, 0]))).tolist()
goal_radius = 0.00001
goal = NavigationGoal(
position=agent_position,
radius=goal_radius
)
episode = NavigationEpisode(
goals=[goal],
episode_id=str(0),
scene_id=scene_mesh_dir,
start_position=agent_position,
start_rotation=agent_rotation,
info={'sound': 'telephone'}
)
episode_sim_config = merge_sim_episode_config(config.TASK_CONFIG.SIMULATOR, episode)
if simulator is None:
simulator = SoundSpaces(episode_sim_config)
simulator.reconfigure(episode_sim_config)
obs, rotation_index = simulator.step(None)
scene_obs[(node, rotation_index)] = obs
num_obs += 1
print('Total number of observations: {}'.format(num_obs))
with open(os.path.join(scene_obs_dir, '{}.pkl'.format(scene)), 'wb') as fo:
pickle.dump(scene_obs, fo)
simulator.close()
del simulator
def reconfigure(self, config: Config) -> None:
self.config = config
if hasattr(self.config.AGENT_0, 'OFFSET'):
self._offset = int(self.config.AGENT_0.OFFSET)
else:
self._offset = 0
if self.config.AUDIO.EVERLASTING:
self._duration = 500
else:
assert hasattr(self.config.AGENT_0, 'DURATION')
self._duration = int(self.config.AGENT_0.DURATION)
self._audio_index = 0
is_same_sound = config.AGENT_0.SOUND_ID == self._current_sound
if not is_same_sound:
self._current_sound = self.config.AGENT_0.SOUND_ID
self._load_single_source_sound()
logging.debug("Switch to sound {} with duration {} seconds".format(self._current_sound, self._duration))
is_same_scene = config.SCENE == self._current_scene
if not is_same_scene:
self._current_scene = config.SCENE
logging.debug('Current scene: {} and sound: {}'.format(self.current_scene_name, self._current_sound))
if self.config.USE_RENDERED_OBSERVATIONS:
with open(self.current_scene_observation_file, 'rb') as fo:
self._frame_cache = pickle.load(fo)
else:
self._sim.close()
del self._sim
self.sim_config = self.create_sim_config(self._sensor_suite)
self._sim = habitat_sim.Simulator(self.sim_config)
self._update_agents_state()
self._frame_cache = dict()
logging.debug('Loaded scene {}'.format(self.current_scene_name))
self.points, self.graph = load_metadata(self.metadata_dir)
for node in self.graph.nodes():
self._position_to_index_mapping[self.position_encoding(self.graph.nodes()[node]['point'])] = node
self._instance2label_mapping = None
if not is_same_scene or not is_same_sound:
self._audiogoal_cache = dict()
self._spectrogram_cache = dict()
self._episode_step_count = 0
# set agent positions
self._receiver_position_index = self._position_to_index(self.config.AGENT_0.START_POSITION)
self._source_position_index = self._position_to_index(self.config.AGENT_0.GOAL_POSITION)
# the agent rotates about +Y starting from -Z counterclockwise,
# so rotation angle 90 means the agent rotate about +Y 90 degrees
self._rotation_angle = int(np.around(np.rad2deg(quat_to_angle_axis(quat_from_coeffs(
self.config.AGENT_0.START_ROTATION))[0]))) % 360
if self.config.USE_RENDERED_OBSERVATIONS:
self._sim.set_agent_state(list(self.graph.nodes[self._receiver_position_index]['point']),
quat_from_coeffs(self.config.AGENT_0.START_ROTATION))
else:
self.set_agent_state(list(self.graph.nodes[self._receiver_position_index]['point']),
self.config.AGENT_0.START_ROTATION)
if self.config.AUDIO.HAS_DISTRACTOR_SOUND:
self._distractor_position_index = self.config.AGENT_0.DISTRACTOR_POSITION_INDEX
self._current_distractor_sound = self.config.AGENT_0.DISTRACTOR_SOUND_ID
self._load_single_distractor_sound()
if self._use_oracle_planner:
self._oracle_actions = self.compute_oracle_actions()
logging.debug("Initial source, agent at: {}, {}, orientation: {}".
format(self._source_position_index, self._receiver_position_index, self.get_orientation()))