def confSim(settings):
sim_config = habitat_sim.SimulatorConfiguration()
sim_config.scene_id = settings["scene"]
#Configuration for Sensors
sensor_spec = []
color_sensor_spec = habitat_sim.SensorSpec()
color_sensor_spec.uuid = "color_sensor"
color_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
color_sensor_spec.resolution = [settings["height"], settings["width"]]
color_sensor_spec.position = [0.0, settings["sensor_height"], 0.0]
sensor_spec.append(color_sensor_spec)
depth_sensor_spec = habitat_sim.SensorSpec()
depth_sensor_spec.uuid = "depth_sensor"
depth_sensor_spec.sensor_type = habitat_sim.SensorType.DEPTH
depth_sensor_spec.resolution = [settings["height"], settings["width"]]
depth_sensor_spec.position = [0.0, settings["sensor_height"], 0.0]
sensor_spec.append(depth_sensor_spec)
#Configure Agent
agent_config = habitat_sim.AgentConfiguration()
agent_config.sensor_specifications = sensor_spec
return habitat_sim.Configuration(sim_config,[agent_config])
def create_sim_config(
self, _sensor_suite: SensorSuite) -> habitat_sim.Configuration:
sim_config = habitat_sim.SimulatorConfiguration()
# sim_config.scene.id = '/private/home/medhini/navigation-analysis-habitat/habitat-api/' + self.config.SCENE
sim_config.scene.id = self.config.SCENE
sim_config.gpu_device_id = self.config.HABITAT_SIM_V0.GPU_DEVICE_ID
agent_config = habitat_sim.AgentConfiguration()
overwrite_config(config_from=self._get_agent_config(),
config_to=agent_config)
sensor_specifications = []
for sensor in _sensor_suite.sensors.values():
sim_sensor_cfg = habitat_sim.SensorSpec()
sim_sensor_cfg.uuid = sensor.uuid
sim_sensor_cfg.resolution = list(
sensor.observation_space.shape[:2])
sim_sensor_cfg.parameters["hfov"] = str(sensor.config.HFOV)
sim_sensor_cfg.position = sensor.config.POSITION
# TODO(maksymets): Add configure method to Sensor API to avoid
# accessing child attributes through parent interface
sim_sensor_cfg.sensor_type = sensor.sim_sensor_type # type: ignore
sensor_specifications.append(sim_sensor_cfg)
# If there is no sensors specified create a dummy sensor so simulator
# won't throw an error
if not _sensor_suite.sensors.values():
sim_sensor_cfg = habitat_sim.SensorSpec()
sim_sensor_cfg.resolution = [1, 1]
sensor_specifications.append(sim_sensor_cfg)
agent_config.sensor_specifications = sensor_specifications
agent_config.action_space = registry.get_action_space_configuration(
self.config.ACTION_SPACE_CONFIG)(self.config).get()
return habitat_sim.Configuration(sim_config, [agent_config])
def setup_agent(dataset_path, random_pos, image_size, focal):
random_up, random_left, random_back = random_pos
backend_cfg = habitat_sim.SimulatorConfiguration()
backend_cfg.scene.id = (dataset_path)
hfov = 2 * np.arctan(0.5 * image_size[1] / focal) / np.pi * 180
# First, let's create a stereo RGB agent
left_rgb_sensor = habitat_sim.SensorSpec()
# Give it the uuid of left_sensor, this will also be how we
# index the observations to retrieve the rendering from this sensor
left_rgb_sensor.uuid = "left_rgb_sensor"
left_rgb_sensor.resolution = image_size
# The left RGB sensor will be 1.5 meters off the ground
# and 0.25 meters to the left of the center of the agent
left_rgb_sensor.position = random_up * habitat_sim.geo.UP + random_left * habitat_sim.geo.LEFT + random_back * habitat_sim.geo.BACK
left_rgb_sensor.parameters["hfov"] = str(hfov)
# Same deal with the right sensor
right_rgb_sensor = habitat_sim.SensorSpec()
right_rgb_sensor.uuid = "right_rgb_sensor"
right_rgb_sensor.resolution = image_size
# The right RGB sensor will be 1.5 meters off the ground
# and 0.25 meters to the right of the center of the agent
right_rgb_sensor.position = left_rgb_sensor.position + 0.5 * habitat_sim.geo.RIGHT
right_rgb_sensor.parameters["hfov"] = str(hfov)
# Now let's do the exact same thing but for a depth camera stereo pair!
left_depth_sensor = habitat_sim.SensorSpec()
left_depth_sensor.uuid = "left_depth_sensor"
left_depth_sensor.resolution = image_size
left_depth_sensor.position = left_rgb_sensor.position
# The only difference is that we set the sensor type to DEPTH
left_depth_sensor.sensor_type = habitat_sim.SensorType.DEPTH
left_depth_sensor.parameters["hfov"] = str(hfov)
right_depth_sensor = habitat_sim.SensorSpec()
right_depth_sensor.uuid = "right_depth_sensor"
right_depth_sensor.resolution = image_size
right_depth_sensor.position = right_rgb_sensor.position
# The only difference is that we set the sensor type to DEPTH
right_depth_sensor.sensor_type = habitat_sim.SensorType.DEPTH
right_depth_sensor.parameters["hfov"] = str(hfov)
# Now we simly set the agent's list of sensor specs to be the two specs for our two sensors
agent_config = habitat_sim.AgentConfiguration()
agent_config.sensor_specifications = [
left_rgb_sensor, right_rgb_sensor, left_depth_sensor,
right_depth_sensor
]
sim = habitat_sim.Simulator(
habitat_sim.Configuration(backend_cfg, [agent_config]))
return sim
def create_sim_config(
self, _sensor_suite: SensorSuite) -> habitat_sim.Configuration:
sim_config = habitat_sim.SimulatorConfiguration()
sim_config.scene.id = self.config.SCENE
sim_config.gpu_device_id = self.config.HABITAT_SIM_V0.GPU_DEVICE_ID
agent_config = habitat_sim.AgentConfiguration()
overwrite_config(config_from=self._get_agent_config(),
config_to=agent_config)
sensor_specifications = []
for sensor in _sensor_suite.sensors.values():
sim_sensor_cfg = habitat_sim.SensorSpec()
sim_sensor_cfg.uuid = sensor.uuid
sim_sensor_cfg.resolution = list(
sensor.observation_space.shape[:2])
sim_sensor_cfg.parameters["hfov"] = str(sensor.config.HFOV)
sim_sensor_cfg.position = sensor.config.POSITION
# TODO(maksymets): Add configure method to Sensor API to avoid
# accessing child attributes through parent interface
sim_sensor_cfg.sensor_type = sensor.sim_sensor_type # type: ignore
sensor_specifications.append(sim_sensor_cfg)
# If there is no sensors specified create a dummy sensor so simulator
# won't throw an error
if not _sensor_suite.sensors.values():
sim_sensor_cfg = habitat_sim.SensorSpec()
sim_sensor_cfg.resolution = [1, 1]
sensor_specifications.append(sim_sensor_cfg)
agent_config.sensor_specifications = sensor_specifications
agent_config.action_space = {
SimulatorActions.LEFT.value:
habitat_sim.ActionSpec(
"turn_left",
habitat_sim.ActuationSpec(amount=self.config.TURN_ANGLE),
),
SimulatorActions.RIGHT.value:
habitat_sim.ActionSpec(
"turn_right",
habitat_sim.ActuationSpec(amount=self.config.TURN_ANGLE),
),
SimulatorActions.FORWARD.value:
habitat_sim.ActionSpec(
"move_forward",
habitat_sim.ActuationSpec(
amount=self.config.FORWARD_STEP_SIZE),
),
SimulatorActions.STOP.value:
habitat_sim.ActionSpec("stop"),
}
return habitat_sim.Configuration(sim_config, [agent_config])
def make_cfg(settings):
sim_cfg = habitat_sim.SimulatorConfiguration()
sim_cfg.gpu_device_id = 0
sim_cfg.scene.id = settings["scene"]
# Note: all sensors must have the same resolution
sensors = {
"color_sensor": {
"sensor_type": habitat_sim.SensorType.COLOR,
"resolution": [settings["height"], settings["width"]],
"position": [0.0, settings["sensor_height"], 0.0],
},
"depth_sensor": {
"sensor_type": habitat_sim.SensorType.DEPTH,
"resolution": [settings["height"], settings["width"]],
"position": [0.0, settings["sensor_height"], 0.0],
},
"semantic_sensor": {
"sensor_type": habitat_sim.SensorType.SEMANTIC,
"resolution": [settings["height"], settings["width"]],
"position": [0.0, settings["sensor_height"], 0.0],
},
}
sensor_specs = []
for sensor_uuid, sensor_params in sensors.items():
if settings[sensor_uuid]:
sensor_spec = habitat_sim.SensorSpec()
sensor_spec.uuid = sensor_uuid
sensor_spec.sensor_type = sensor_params["sensor_type"]
sensor_spec.resolution = sensor_params["resolution"]
sensor_spec.position = sensor_params["position"]
sensor_specs.append(sensor_spec)
# Here you can specify the amount of displacement in a forward action and the turn angle
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
agent_cfg.action_space = {
"move_forward":
habitat_sim.agent.ActionSpec(
"move_forward", habitat_sim.agent.ActuationSpec(amount=0.05)),
"move_back":
habitat_sim.agent.ActionSpec(
"move_forward", habitat_sim.agent.ActuationSpec(amount=-0.05)),
"turn_left":
habitat_sim.agent.ActionSpec(
"turn_left", habitat_sim.agent.ActuationSpec(amount=2.0)),
"turn_right":
habitat_sim.agent.ActionSpec(
"turn_right", habitat_sim.agent.ActuationSpec(amount=2.0)),
"look_up":
habitat_sim.ActionSpec("look_up",
habitat_sim.ActuationSpec(amount=2.0)),
"look_down":
habitat_sim.ActionSpec("look_down",
habitat_sim.ActuationSpec(amount=2.0))
}
return habitat_sim.Configuration(sim_cfg, [agent_cfg])
def make_cfg(settings, intrinsics=None):
sim_cfg = habitat_sim.SimulatorConfiguration()
sim_cfg.gpu_device_id = 0
sim_cfg.scene_id = settings["scene"]
if intrinsics is None:
hfov = 63.414969 # use suncg
else:
hfov = get_hfov(intrinsics, settings['width'])
# Note: all sensors must have the same resolution
sensors = {
"color_sensor": {
"sensor_type": habitat_sim.SensorType.COLOR,
"resolution": [settings["height"], settings["width"]],
"position": [0.0, settings["sensor_height"], 0.0],
"hfov": str(hfov)
},
"depth_sensor": {
"sensor_type": habitat_sim.SensorType.DEPTH,
"resolution": [settings["height"], settings["width"]],
"position": [0.0, settings["sensor_height"], 0.0],
"hfov": str(hfov)
},
"semantic_sensor": {
"sensor_type": habitat_sim.SensorType.SEMANTIC,
"resolution": [settings["height"], settings["width"]],
"position": [0.0, settings["sensor_height"], 0.0],
"hfov": str(hfov)
},
}
sensor_specs = []
for sensor_uuid, sensor_params in sensors.items():
if settings[sensor_uuid]:
sensor_spec = habitat_sim.SensorSpec()
sensor_spec.uuid = sensor_uuid
sensor_spec.sensor_type = sensor_params["sensor_type"]
sensor_spec.resolution = sensor_params["resolution"]
sensor_spec.position = sensor_params["position"]
sensor_spec.parameters['hfov'] = sensor_params["hfov"]
sensor_specs.append(sensor_spec)
# Here you can specify the amount of displacement in a forward action and the turn angle
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
agent_cfg.action_space = {
"move_forward":
habitat_sim.agent.ActionSpec(
"move_forward", habitat_sim.agent.ActuationSpec(amount=0.25)),
"turn_left":
habitat_sim.agent.ActionSpec(
"turn_left", habitat_sim.agent.ActuationSpec(amount=30.0)),
"turn_right":
habitat_sim.agent.ActionSpec(
"turn_right", habitat_sim.agent.ActuationSpec(amount=30.0)),
}
return habitat_sim.Configuration(sim_cfg, [agent_cfg])
def create_sim_config(
self, _sensor_suite: SensorSuite) -> habitat_sim.Configuration:
sim_config = habitat_sim.SimulatorConfiguration()
sim_config.scene.id = self.config.SCENE
sim_config.gpu_device_id = self.config.HABITAT_SIM_V0.GPU_DEVICE_ID
sim_config.allow_sliding = self.config.HABITAT_SIM_V0.ALLOW_SLIDING
sim_config.enable_physics = self.config.HABITAT_SIM_V0.ENABLE_PHYSICS
sim_config.physics_config_file = (
self.config.HABITAT_SIM_V0.PHYSICS_CONFIG_FILE)
agent_config = habitat_sim.AgentConfiguration()
overwrite_config(config_from=self._get_agent_config(),
config_to=agent_config)
sensor_specifications = []
for sensor in _sensor_suite.sensors.values():
sim_sensor_cfg = habitat_sim.SensorSpec()
sim_sensor_cfg.uuid = sensor.uuid
sim_sensor_cfg.resolution = list(
sensor.observation_space.shape[:2])
sim_sensor_cfg.parameters["hfov"] = str(sensor.config.HFOV)
sim_sensor_cfg.position = sensor.config.POSITION
# TODO(maksymets): Add configure method to Sensor API to avoid
# accessing child attributes through parent interface
sim_sensor_cfg.sensor_type = sensor.sim_sensor_type # type: ignore
sim_sensor_cfg.gpu2gpu_transfer = (
self.config.HABITAT_SIM_V0.GPU_GPU)
sensor_specifications.append(sim_sensor_cfg)
agent_config.sensor_specifications = sensor_specifications
agent_config.action_space = registry.get_action_space_configuration(
self.config.ACTION_SPACE_CONFIG)(self.config).get()
return habitat_sim.Configuration(sim_config, [agent_config])
def create_sim_config(
self, _sensor_suite: SensorSuite) -> habitat_sim.Configuration:
sim_config = habitat_sim.SimulatorConfiguration()
overwrite_config(config_from=self.config.HABITAT_SIM_V0,
config_to=sim_config)
sim_config.scene.id = self.config.SCENE
agent_config = habitat_sim.AgentConfiguration()
overwrite_config(config_from=self._get_agent_config(),
config_to=agent_config)
sensor_specifications = []
for sensor in _sensor_suite.sensors.values():
sim_sensor_cfg = habitat_sim.SensorSpec()
overwrite_config(config_from=sensor.config,
config_to=sim_sensor_cfg)
sim_sensor_cfg.uuid = sensor.uuid
sim_sensor_cfg.resolution = list(
sensor.observation_space.shape[:2])
sim_sensor_cfg.parameters["hfov"] = str(sensor.config.HFOV)
# TODO(maksymets): Add configure method to Sensor API to avoid
# accessing child attributes through parent interface
sim_sensor_cfg.sensor_type = sensor.sim_sensor_type # type: ignore
sim_sensor_cfg.gpu2gpu_transfer = (
self.config.HABITAT_SIM_V0.GPU_GPU)
sensor_specifications.append(sim_sensor_cfg)
agent_config.sensor_specifications = sensor_specifications
agent_config.action_space = registry.get_action_space_configuration(
self.config.ACTION_SPACE_CONFIG)(self.config).get()
return habitat_sim.Configuration(sim_config, [agent_config])
def create_sensor(orientation=[0.0, 0.0, 0.0],
position=[0.0, 0.0, 0.0],
sensor_resolution=[512, 512],
sensor_uuid="my_sensor",
camera_type="C"):
"""
:param orientation: Axis Angle representation 3 values and describes the rotation with respect to Habitat frame
NOT agent frame
:param position: in meters x - distance from the ground y - right of the agent, z is up
:param sensor_resolution: w has to equal to h for foveation (Ryan makes this assumption for shader)
:param sensor_uuid: The identifier that will be used to refer to the sensor
:param camera_type: camera_type "C" is RGB Camera and "D" is Depth
:return: sensor object
The sensor to the right is the left eye for OREO
"""
new_sensor = habitat_sim.SensorSpec()
new_sensor.uuid = sensor_uuid
if camera_type == "D":
new_sensor.sensor_type = habitat_sim.SensorType.DEPTH
else:
new_sensor.sensor_type = habitat_sim.SensorType.COLOR
new_sensor.resolution = sensor_resolution
new_sensor.position = position # +ive x is to the right, +ive y is UP, -ive z is in front
new_sensor.orientation = np.array(orientation, dtype=float)
return new_sensor
def make_cfg(settings):
sim_cfg = habitat_sim.SimulatorConfiguration()
sim_cfg.gpu_device_id = 0
sim_cfg.scene_id = settings["scene"]
sim_cfg.enable_physics = settings["enable_physics"]
# Note: all sensors must have the same resolution
sensors = {
"color_sensor_1st_person": {
"sensor_type": habitat_sim.SensorType.COLOR,
"resolution": [settings["height"], settings["width"]],
"position": [0.0, settings["sensor_height"], 0.0],
"orientation": [settings["sensor_pitch"], 0.0, 0.0],
},
"depth_sensor_1st_person": {
"sensor_type": habitat_sim.SensorType.DEPTH,
"resolution": [settings["height"], settings["width"]],
"position": [0.0, settings["sensor_height"], 0.0],
"orientation": [settings["sensor_pitch"], 0.0, 0.0],
},
"semantic_sensor_1st_person": {
"sensor_type": habitat_sim.SensorType.SEMANTIC,
"resolution": [settings["height"], settings["width"]],
"position": [0.0, settings["sensor_height"], 0.0],
"orientation": [settings["sensor_pitch"], 0.0, 0.0],
},
# configure the 3rd person cam specifically:
"color_sensor_3rd_person": {
"sensor_type": habitat_sim.SensorType.COLOR,
"resolution": [settings["height"], settings["width"]],
"position": [0.0, settings["sensor_height"] + 0.2, 0.2],
"orientation": np.array([-math.pi / 4, 0, 0]),
},
}
sensor_specs = []
for sensor_uuid, sensor_params in sensors.items():
if settings[sensor_uuid]:
sensor_spec = habitat_sim.SensorSpec()
sensor_spec.uuid = sensor_uuid
sensor_spec.sensor_type = sensor_params["sensor_type"]
sensor_spec.resolution = sensor_params["resolution"]
sensor_spec.position = sensor_params["position"]
sensor_spec.orientation = sensor_params["orientation"]
sensor_specs.append(sensor_spec)
# Here you can specify the amount of displacement in a forward action and the turn angle
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
return habitat_sim.Configuration(sim_cfg, [agent_cfg])
def make_configuration():
# simulator configuration
backend_cfg = habitat_sim.SimulatorConfiguration()
backend_cfg.scene.id = "data/scene_datasets/habitat-test-scenes/van-gogh-room.glb"
backend_cfg.enable_physics = True
# agent configuration
sensor_cfg = habitat_sim.SensorSpec()
sensor_cfg.resolution = [1080, 960]
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = [sensor_cfg]
return habitat_sim.Configuration(backend_cfg, [agent_cfg])
def create_test_sensor(sensor_uuid="my_sensor", camera_type="C"):
"""orientation - Axis Angle representation
camera_type "C" is RGB Camera and "D" is Depth."""
new_sensor = habitat_sim.SensorSpec()
new_sensor.uuid = sensor_uuid
if camera_type == "D":
new_sensor.sensor_type = habitat_sim.SensorType.DEPTH
else:
new_sensor.sensor_type = habitat_sim.SensorType.COLOR
new_sensor.resolution = [512, 512]
return new_sensor
def make_configuration():
# simulator configuration
backend_cfg = habitat_sim.SimulatorConfiguration()
backend_cfg.scene_id = os.path.join(
data_path, "scene_datasets/mp3d/x8F5xyUWy9e/x8F5xyUWy9e.glb"
)
assert os.path.exists(backend_cfg.scene_id)
backend_cfg.enable_physics = True
# sensor configurations
# Note: all sensors must have the same resolution
# setup 2 rgb sensors for 1st and 3rd person views
camera_resolution = [544, 720]
sensors = {
"rgba_camera_1stperson": {
"sensor_type": habitat_sim.SensorType.COLOR,
"resolution": camera_resolution,
"position": [0.0, 0.6, 0.0],
"orientation": [0.0, 0.0, 0.0],
},
"depth_camera_1stperson": {
"sensor_type": habitat_sim.SensorType.DEPTH,
"resolution": camera_resolution,
"position": [0.0, 0.6, 0.0],
"orientation": [0.0, 0.0, 0.0],
},
"semantic_camera_1stperson": {
"sensor_type": habitat_sim.SensorType.SEMANTIC,
"resolution": camera_resolution,
"position": [0.0, 1.0, 0.3],
"orientation": [-45, 0.0, 0.0],
},
}
sensor_specs = []
for sensor_uuid, sensor_params in sensors.items():
sensor_spec = habitat_sim.SensorSpec()
sensor_spec.uuid = sensor_uuid
sensor_spec.sensor_type = sensor_params["sensor_type"]
sensor_spec.resolution = sensor_params["resolution"]
sensor_spec.position = sensor_params["position"]
sensor_spec.orientation = sensor_params["orientation"]
sensor_specs.append(sensor_spec)
# agent configuration
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
return habitat_sim.Configuration(backend_cfg, [agent_cfg])
def _make_cfg(self, settings):
sim_cfg = habitat_sim.SimulatorConfiguration()
sim_cfg.gpu_device_id = int(self.params.get('gpu',
_DEFAULT_GPU_DEVICE))
sim_cfg.scene.id = settings["scene"]
# Setup Sensors
# ------------------------------------------------------------
# Note: all sensors must have the same resolution
sensors = {
"color_sensor": {
"sensor_type": habitat_sim.SensorType.COLOR,
"resolution": [settings["height"], settings["width"]],
"position": [0.0, settings["sensor_height"], 0.0],
},
"semantic_sensor": {
"sensor_type": habitat_sim.SensorType.SEMANTIC,
"resolution": [settings["height"], settings["width"]],
"position": [0.0, settings["sensor_height"], 0.0],
}
}
if self.sim_settings['depth_sensor']:
sensors["depth_sensor"] = {
"sensor_type": habitat_sim.SensorType.DEPTH,
"resolution": [settings["height"], settings["width"]],
"position": [0.0, settings["sensor_height"], 0.0],
}
sensor_specs = []
for sensor_uuid, sensor_params in sensors.items():
if settings[sensor_uuid]:
sensor_spec = habitat_sim.SensorSpec()
sensor_spec.uuid = sensor_uuid
sensor_spec.sensor_type = sensor_params["sensor_type"]
sensor_spec.resolution = sensor_params["resolution"]
sensor_spec.position = sensor_params["position"]
sensor_spec.parameters["hfov"] = str(settings["hfov"])
sensor_specs.append(sensor_spec)
# ------------------------------------------------------------
# Setup Agent
# ------------------------------------------------------------
agent_cfg = habitat_sim.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
agent_cfg.action_space = {}
# ------------------------------------------------------------
return habitat_sim.Configuration(sim_cfg, [agent_cfg])
def create_sim_config(
self, _sensor_suite: SensorSuite
) -> habitat_sim.Configuration:
sim_config = habitat_sim.SimulatorConfiguration()
# Check if Habitat-Sim is post Scene Config Update
if not hasattr(sim_config, "scene_id"):
raise RuntimeError(
"Incompatible version of Habitat-Sim detected, please upgrade habitat_sim"
)
overwrite_config(
config_from=self.habitat_config.HABITAT_SIM_V0,
config_to=sim_config,
)
sim_config.scene_id = self.habitat_config.SCENE
agent_config = habitat_sim.AgentConfiguration()
overwrite_config(
config_from=self._get_agent_config(), config_to=agent_config
)
sensor_specifications = []
for sensor in _sensor_suite.sensors.values():
sim_sensor_cfg = habitat_sim.SensorSpec()
overwrite_config(
config_from=sensor.config, config_to=sim_sensor_cfg
)
sim_sensor_cfg.uuid = sensor.uuid
sim_sensor_cfg.resolution = list(
sensor.observation_space.shape[:2]
)
sim_sensor_cfg.parameters["hfov"] = str(sensor.config.HFOV)
# TODO(maksymets): Add configure method to Sensor API to avoid
# accessing child attributes through parent interface
# We know that the Sensor has to be one of these Sensors
sensor = cast(HabitatSimVizSensors, sensor)
sim_sensor_cfg.sensor_type = sensor.sim_sensor_type
sim_sensor_cfg.gpu2gpu_transfer = (
self.habitat_config.HABITAT_SIM_V0.GPU_GPU
)
sensor_specifications.append(sim_sensor_cfg)
agent_config.sensor_specifications = sensor_specifications
agent_config.action_space = registry.get_action_space_configuration(
self.habitat_config.ACTION_SPACE_CONFIG
)(self.habitat_config).get()
return habitat_sim.Configuration(sim_config, [agent_config])
def make_configuration():
# simulator configuration
backend_cfg = habitat_sim.SimulatorConfiguration()
backend_cfg.scene.id = "data/scene_datasets/habitat-test-scenes/apartment_1.glb"
backend_cfg.enable_physics = True
# sensor configurations
# Note: all sensors must have the same resolution
# setup 2 rgb sensors for 1st and 3rd person views
camera_resolution = [540, 720]
sensors = {
"rgba_camera_1stperson": {
"sensor_type": habitat_sim.SensorType.COLOR,
"resolution": camera_resolution,
"position": [0.0, 0.6, 0.0],
"orientation": [0.0, 0.0, 0.0],
},
"depth_camera_1stperson": {
"sensor_type": habitat_sim.SensorType.DEPTH,
"resolution": camera_resolution,
"position": [0.0, 0.6, 0.0],
"orientation": [0.0, 0.0, 0.0],
},
"rgba_camera_3rdperson": {
"sensor_type": habitat_sim.SensorType.COLOR,
"resolution": camera_resolution,
"position": [0.0, 1.0, 0.3],
"orientation": [-45, 0.0, 0.0],
},
}
sensor_specs = []
for sensor_uuid, sensor_params in sensors.items():
sensor_spec = habitat_sim.SensorSpec()
sensor_spec.uuid = sensor_uuid
sensor_spec.sensor_type = sensor_params["sensor_type"]
sensor_spec.resolution = sensor_params["resolution"]
sensor_spec.position = sensor_params["position"]
sensor_spec.orientation = sensor_params["orientation"]
sensor_specs.append(sensor_spec)
# agent configuration
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
return habitat_sim.Configuration(backend_cfg, [agent_cfg])
def make_configuration():
# simulator configuration
backend_cfg = habitat_sim.SimulatorConfiguration()
backend_cfg.scene_id = os.path.join(
data_path, "scene_datasets/habitat-test-scenes/apartment_1.glb")
assert os.path.exists(backend_cfg.scene_id)
backend_cfg.enable_physics = True
# Enable gfx replay save. See also our call to sim.gfx_replay_manager.save_keyframe()
# below.
backend_cfg.enable_gfx_replay_save = True
sensor_cfg = habitat_sim.SensorSpec()
sensor_cfg.resolution = [544, 720]
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = [sensor_cfg]
return habitat_sim.Configuration(backend_cfg, [agent_cfg])
def create_sensor(orientation, position=[0.0,0.0,0.0], sensor_resolution=[512, 512],
sensor_uuid="my_sensor", camera_type="C"):
"""orientation - Axis Angle representation 3 values and describes the rotation with respect to agent.
camera_type "C" is RGB Camera and "D" is Depth."""
new_sensor = habitat_sim.SensorSpec()
new_sensor.uuid = sensor_uuid
if camera_type == "D":
new_sensor.sensor_type = habitat_sim.SensorType.DEPTH
else:
new_sensor.sensor_type = habitat_sim.SensorType.COLOR
new_sensor.resolution = sensor_resolution
new_sensor.position = position # +ive x is to the right, +ive y is UP, -ive z is in front
new_sensor.orientation = np.array(orientation,dtype=float)
return new_sensor
def make_simple_cfg(settings):
# simulator backend
sim_cfg = habitat_sim.SimulatorConfiguration()
sim_cfg.scene.id = settings["scene"]
# agent
agent_cfg = habitat_sim.agent.AgentConfiguration()
# In the 1st example, we attach only one sensor,
# a RGB visual sensor, to the agent
rgb_sensor_spec = habitat_sim.SensorSpec()
rgb_sensor_spec.uuid = "color_sensor"
rgb_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
rgb_sensor_spec.resolution = [settings["height"], settings["width"]]
rgb_sensor_spec.position = [0.0, settings["sensor_height"], 0.0]
agent_cfg.sensor_specifications = [rgb_sensor_spec]
return habitat_sim.Configuration(sim_cfg, [agent_cfg])
def configure_sim(self, sim_settings):
# Configure simulator with a scene and robot
sim_cfg = habitat_sim.SimulatorConfiguration()
sim_cfg.scene.id = self.sim_settings["scene"]
# Robot config
agent_cfg = habitat_sim.agent.AgentConfiguration()
# Add RGB sensor to agent (camera)
rgb_sensor_spec = habitat_sim.SensorSpec()
rgb_sensor_spec.uuid = "color_sensor"
rgb_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
rgb_sensor_spec.resolution = [
self.sim_settings["rgb_res_height"],
self.sim_settings["rgb_res_width"]
]
rgb_sensor_spec.position = [0.0, self.sim_settings["rgb_height"], 0.0]
agent_cfg.sensor_specifications = [rgb_sensor_spec]
return habitat_sim.Configuration(sim_cfg, [agent_cfg])
def make_configuration():
# simulator configuration
backend_cfg = habitat_sim.SimulatorConfiguration()
backend_cfg.scene_id = "NONE"
backend_cfg.enable_physics = True
# sensor configurations
sensor_specs = []
sensor_spec = habitat_sim.SensorSpec()
sensor_spec.uuid = "rgba_camera_1stperson"
sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
sensor_spec.resolution = [544, 720]
sensor_specs.append(sensor_spec)
# agent configuration
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
return habitat_sim.Configuration(backend_cfg, [agent_cfg])
def make_configuration(scene_file):
# simulator configuration
backend_cfg = habitat_sim.SimulatorConfiguration()
backend_cfg.scene.id = scene_file
backend_cfg.enable_physics = True
# sensor configurations
# Note: all sensors must have the same resolution
# setup rgb and semantic sensors
camera_resolution = [1080, 960]
sensors = {
"rgba_camera": {
"sensor_type": habitat_sim.SensorType.COLOR,
"resolution": camera_resolution,
"position": [0.0, 1.5, 0.0], # ::: fix y to be 0 later
},
"semantic_camera": {
"sensor_type": habitat_sim.SensorType.SEMANTIC,
"resolution": camera_resolution,
"position": [0.0, 1.5, 0.0],
},
}
sensor_specs = []
for sensor_uuid, sensor_params in sensors.items():
sensor_spec = habitat_sim.SensorSpec()
sensor_spec.uuid = sensor_uuid
sensor_spec.sensor_type = sensor_params["sensor_type"]
sensor_spec.resolution = sensor_params["resolution"]
sensor_spec.position = sensor_params["position"]
sensor_specs.append(sensor_spec)
# agent configuration
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
return habitat_sim.Configuration(backend_cfg, [agent_cfg])
def RunSimulation(envNum, record, path, Coords, seed):
settings = {"image_height": 720, "image_width": 1080, "random_seed": seed}
# pick a scene file to load
scene_file = path
if record:
os.system("mkdir CapturedFrames")
os.system(("mkdir CapturedFrames/Environment" + str(envNum)))
# --------------------------
# Simulator Setup
# --------------------------
# create a SimulatorConfiguration object
sim_cfg = habitat_sim.SimulatorConfiguration()
sim_cfg.scene.id = scene_file
# configure the simulator to intialize a PhysicsManager
sim_cfg.enable_physics = True
# configure the simulator to load a specific simulator configuration file (define paths you your object models here)
sim_cfg.physics_config_file = "data/default.phys_scene_config.json"
# pick this based on your device
relative_camera_position = [0.0, 0, 0.0]
# Note: all sensors must have the same resolution
sensors = {
"color": {
"sensor_type": habitat_sim.SensorType.COLOR,
"resolution": [settings["image_height"], settings["image_width"]],
"position": relative_camera_position,
},
"depth": {
"sensor_type": habitat_sim.SensorType.DEPTH,
"resolution": [settings["image_height"], settings["image_width"]],
"position": relative_camera_position,
},
}
sensor_specs = []
for sensor_uuid, sensor_params in sensors.items():
sensor_spec = habitat_sim.SensorSpec()
sensor_spec.uuid = sensor_uuid
sensor_spec.sensor_type = sensor_params["sensor_type"]
sensor_spec.resolution = sensor_params["resolution"]
sensor_spec.position = sensor_params["position"]
sensor_specs.append(sensor_spec)
# setup the agent/camera
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
agent_cfg.action_space = {
"move_forward":
habitat_sim.agent.ActionSpec(
"move_forward", habitat_sim.agent.ActuationSpec(amount=0.2)),
"turn_left":
habitat_sim.agent.ActionSpec(
"turn_left", habitat_sim.agent.ActuationSpec(amount=10.0)),
}
# generate the full simulator configuration
combined_cfg = habitat_sim.Configuration(sim_cfg, [agent_cfg])
# initialize the simulator
sim = habitat_sim.Simulator(combined_cfg)
sim.agents[0].controls.move_filter_fn = sim._scene_collision_step_filter
# set random seed
random.seed(settings["random_seed"])
sim.seed(settings["random_seed"])
# initialize the agent state
sim.initialize_agent(0)
# ----------------------------------
# Quadrotor Prototype Functionality
# ----------------------------------
# assuming the first entry in the object list corresponds to the quadrotor model
quadrotor_id = sim.add_object(0)
# the object will not be affected by forces such as gravity and will remain static until explicitly moved
sim.set_object_motion_type(habitat_sim.physics.MotionType.KINEMATIC,
quadrotor_id)
## Randomly initialize the Ball
initial_height = 0.00001
possible_initial_point = sim.pathfinder.get_random_navigable_point(
) + np.array([0, initial_height, 0])
sim.set_translation(possible_initial_point, quadrotor_id)
while (sim.contact_test(quadrotor_id)):
possible_initial_point = sim.pathfinder.get_random_navigable_point(
) + np.array([0, initial_height, 0])
sim.set_translation(possible_initial_point, quadrotor_id)
# place the object in the air
# sim.set_translation(np.array([-0.569043, 2.04804, 13.6156]), quadrotor_id)
sim.agents[0].scene_node.translation = sim.get_translation(quadrotor_id)
# static_3rd_person_camera_position = np.array([-0.569043, 2.04804, 12.6156])
static_3rd_person_camera_position = np.array(
sim.get_translation(quadrotor_id)) - np.array([0, 0, 2])
initTranslation = sim.get_translation(quadrotor_id)
for frame in range(len(Coords)):
dx = Coords[frame][0]
dy = Coords[frame][1]
dz = 0
dispInFrame = np.array([dx, dy, dz])
currentTranslation = np.array(sim.get_translation(quadrotor_id))
sim.set_translation(dispInFrame + currentTranslation, quadrotor_id)
if (sim.contact_test(quadrotor_id)):
print("Collided at frame : ", frame)
break
if record:
# get/save 1st person images
# agent_observations = sim.get_sensor_observations()
#save_observation(agent_observations, frame, "agent_")
# move camera to 3rd person view
sim.get_agent(0).scene_node.translation = np.array(
static_3rd_person_camera_position)
sim.get_agent(0).scene_node.rotation = ut.quat_to_magnum(
ut.quat_look_at(sim.get_translation(quadrotor_id),
static_3rd_person_camera_position))
agent_observations = sim.get_sensor_observations()
save_observation(agent_observations, frame, envNum, "3rdperson_")
# reset the agent for the next step
sim.get_agent(0).scene_node.translation = sim.get_translation(
quadrotor_id)
sim.get_agent(0).scene_node.rotation = sim.get_rotation(
quadrotor_id)
if record:
MakeVideo(envNum)
sim.close()
del sim
def make_cfg(settings):
sim_cfg = habitat_sim.SimulatorConfiguration()
if "frustum_culling" in settings:
sim_cfg.frustum_culling = settings["frustum_culling"]
else:
sim_cfg.frustum_culling = False
if "enable_physics" in settings:
sim_cfg.enable_physics = settings["enable_physics"]
if "physics_config_file" in settings:
sim_cfg.physics_config_file = settings["physics_config_file"]
if not settings["silent"]:
print("sim_cfg.physics_config_file = " + sim_cfg.physics_config_file)
if "scene_light_setup" in settings:
sim_cfg.scene_light_setup = settings["scene_light_setup"]
sim_cfg.gpu_device_id = 0
if not hasattr(sim_cfg, "scene_id"):
raise RuntimeError(
"Error: Please upgrade habitat-sim. SimulatorConfig API version mismatch"
)
sim_cfg.scene_id = settings["scene"]
# define default sensor parameters (see src/esp/Sensor/Sensor.h)
sensors = {
"color_sensor": { # active if sim_settings["color_sensor"]
"sensor_type": habitat_sim.SensorType.COLOR,
"resolution": [settings["height"], settings["width"]],
"position": [0.0, settings["sensor_height"], 0.0],
"sensor_subtype": habitat_sim.SensorSubType.PINHOLE,
},
"depth_sensor": { # active if sim_settings["depth_sensor"]
"sensor_type": habitat_sim.SensorType.DEPTH,
"resolution": [settings["height"], settings["width"]],
"position": [0.0, settings["sensor_height"], 0.0],
"sensor_subtype": habitat_sim.SensorSubType.PINHOLE,
},
"semantic_sensor": { # active if sim_settings["semantic_sensor"]
"sensor_type": habitat_sim.SensorType.SEMANTIC,
"resolution": [settings["height"], settings["width"]],
"position": [0.0, settings["sensor_height"], 0.0],
"sensor_subtype": habitat_sim.SensorSubType.PINHOLE,
},
"ortho_sensor": { # active if sim_settings["ortho_sensor"]
"sensor_type": habitat_sim.SensorType.COLOR,
"resolution": [settings["height"], settings["width"]],
"position": [0.0, settings["sensor_height"], 0.0],
"sensor_subtype": habitat_sim.SensorSubType.ORTHOGRAPHIC,
},
}
# create sensor specifications
sensor_specs = []
for sensor_uuid, sensor_params in sensors.items():
if settings[sensor_uuid]:
sensor_spec = habitat_sim.SensorSpec()
sensor_spec.uuid = sensor_uuid
sensor_spec.sensor_type = sensor_params["sensor_type"]
sensor_spec.sensor_subtype = sensor_params["sensor_subtype"]
sensor_spec.resolution = sensor_params["resolution"]
sensor_spec.position = sensor_params["position"]
sensor_spec.gpu2gpu_transfer = False
if not settings["silent"]:
print("==== Initialized Sensor Spec: =====")
print("Sensor uuid: ", sensor_spec.uuid)
print("Sensor type: ", sensor_spec.sensor_type)
print("Sensor position: ", sensor_spec.position)
print("===================================")
sensor_specs.append(sensor_spec)
# create agent specifications
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
agent_cfg.action_space = {
"move_forward": habitat_sim.agent.ActionSpec(
"move_forward", habitat_sim.agent.ActuationSpec(amount=0.25)
),
"turn_left": habitat_sim.agent.ActionSpec(
"turn_left", habitat_sim.agent.ActuationSpec(amount=10.0)
),
"turn_right": habitat_sim.agent.ActionSpec(
"turn_right", habitat_sim.agent.ActuationSpec(amount=10.0)
),
}
# override action space to no-op to test physics
if sim_cfg.enable_physics:
agent_cfg.action_space = {
"move_forward": habitat_sim.agent.ActionSpec(
"move_forward", habitat_sim.agent.ActuationSpec(amount=0.0)
)
}
return habitat_sim.Configuration(sim_cfg, [agent_cfg])
def create_sim_config(
self, _sensor_suite: SensorSuite) -> habitat_sim.Configuration:
sim_config = habitat_sim.SimulatorConfiguration()
# Check if Habitat-Sim is post Scene Config Update
if not hasattr(sim_config, "scene_id"):
raise RuntimeError(
"Incompatible version of Habitat-Sim detected, please upgrade habitat_sim"
)
overwrite_config(
config_from=self.habitat_config.HABITAT_SIM_V0,
config_to=sim_config,
# Ignore key as it gets propogated to sensor below
ignore_keys={"gpu_gpu"},
)
sim_config.scene_id = self.habitat_config.SCENE
agent_config = habitat_sim.AgentConfiguration()
overwrite_config(
config_from=self._get_agent_config(),
config_to=agent_config,
# These keys are only used by Hab-Lab
ignore_keys={
"is_set_start_state",
# This is the Sensor Config. Unpacked below
"sensors",
"start_position",
"start_rotation",
},
)
sensor_specifications = []
for sensor in _sensor_suite.sensors.values():
sim_sensor_cfg = habitat_sim.SensorSpec()
# TODO Handle configs for custom VisualSensors that might need
# their own ignore_keys. Maybe with special key / checking
# SensorType
overwrite_config(
config_from=sensor.config,
config_to=sim_sensor_cfg,
# These keys are only used by Hab-Lab
# or translated into the sensor config manually
ignore_keys={
"height",
"hfov",
"max_depth",
"min_depth",
"normalize_depth",
"type",
"width",
},
)
sim_sensor_cfg.uuid = sensor.uuid
sim_sensor_cfg.resolution = list(
sensor.observation_space.shape[:2])
sim_sensor_cfg.parameters["hfov"] = str(sensor.config.HFOV)
# TODO(maksymets): Add configure method to Sensor API to avoid
# accessing child attributes through parent interface
# We know that the Sensor has to be one of these Sensors
sensor = cast(HabitatSimVizSensors, sensor)
sim_sensor_cfg.sensor_type = sensor.sim_sensor_type
sim_sensor_cfg.gpu2gpu_transfer = (
self.habitat_config.HABITAT_SIM_V0.GPU_GPU)
sensor_specifications.append(sim_sensor_cfg)
agent_config.sensor_specifications = sensor_specifications
agent_config.action_space = registry.get_action_space_configuration(
self.habitat_config.ACTION_SPACE_CONFIG)(
self.habitat_config).get()
return habitat_sim.Configuration(sim_config, [agent_config])
def setupAgentwithSensors(display=True):
global cv2
# Only import cv2 if we are doing to display
if display:
import cv2 as _cv2
cv2 = _cv2
cv2.namedWindow("stereo_pair")
# cv2.namedWindow("depth_pair")
backend_cfg = habitat_sim.SimulatorConfiguration()
backend_cfg.scene.id = (
"data/scene_datasets/habitat-test-scenes/skokloster-castle.glb"
) # data/scene_datasets/habitat-test-scenes/skokloster-castle.glb
# cam_baseline = 0.2
# cam_focalLength = 450
# height = 640
# width = 960
# constructPyramid()
hfov = 2 * math.degrees(math.atan(width / (2 * cam_focalLength)))
vfov = 2 * math.degrees(math.atan(height / (2 * cam_focalLength)))
# First, let's create a stereo RGB agent
left_rgb_sensor = habitat_sim.SensorSpec()
# Give it the uuid of left_sensor, this will also be how we
# index the observations to retrieve the rendering from this sensor
left_rgb_sensor.uuid = "left_sensor"
left_rgb_sensor.resolution = [height, width]
left_rgb_sensor.parameters["hfov"] = str(hfov)
left_rgb_sensor.parameters["vfov"] = str(vfov)
# The left RGB sensor will be 1.5 meters off the ground
# and 0.25 meters to the left of the center of the agent
left_rgb_sensor.position = 1.5 * habitat_sim.geo.UP + (
cam_baseline / 2) * habitat_sim.geo.LEFT
# Same deal with the right sensor
right_rgb_sensor = habitat_sim.SensorSpec()
right_rgb_sensor.uuid = "right_sensor"
right_rgb_sensor.resolution = [height, width]
right_rgb_sensor.parameters["hfov"] = str(hfov)
right_rgb_sensor.parameters["vfov"] = str(vfov)
# The right RGB sensor will be 1.5 meters off the ground
# and 0.25 meters to the right of the center of the agent
right_rgb_sensor.position = 1.5 * habitat_sim.geo.UP + (
cam_baseline / 2) * habitat_sim.geo.RIGHT
# Now let's do the exact same thing but for a depth camera stereo pair!
left_depth_sensor = habitat_sim.SensorSpec()
left_depth_sensor.uuid = "left_sensor_depth"
left_depth_sensor.resolution = [height, width]
left_depth_sensor.parameters["hfov"] = str(hfov)
left_depth_sensor.parameters["vfov"] = str(vfov)
left_depth_sensor.position = 1.5 * habitat_sim.geo.UP + (
cam_baseline / 2) * habitat_sim.geo.LEFT
# The only difference is that we set the sensor type to DEPTH
left_depth_sensor.sensor_type = habitat_sim.SensorType.DEPTH
right_depth_sensor = habitat_sim.SensorSpec()
right_depth_sensor.uuid = "right_sensor_depth"
right_depth_sensor.resolution = [height, width]
right_depth_sensor.parameters["hfov"] = str(hfov)
right_depth_sensor.parameters["vfov"] = str(vfov)
right_depth_sensor.position = 1.5 * habitat_sim.geo.UP + (
cam_baseline / 2) * habitat_sim.geo.RIGHT
# The only difference is that we set the sensor type to DEPTH
right_depth_sensor.sensor_type = habitat_sim.SensorType.DEPTH
agent_config = habitat_sim.AgentConfiguration(
) # set configuration for agent (id = 0)
agent_config.sensor_specifications = [
left_rgb_sensor, right_rgb_sensor, left_depth_sensor,
right_depth_sensor
]
sim = habitat_sim.simulator.Simulator(
habitat_sim.Configuration(backend_cfg, [agent_config]))
# set agent position
inital_state_agent = place_agent(sim, agent_config)
# print("inital agent state: {0} ".format(inital_state_agent))
print("inital agent's sensor state: {0} ".format(
inital_state_agent.sensor_states))
# set action
_render(sim, isdepth=True)
STEP = 0.1
Width = 1280
Height = 1080
Baseline = 0.2
Cam_H = 1.5
HFOV = np.pi / 2.0
fx = 1 / np.tan(HFOV / 2.0) * Width / 2.0
BF = Baseline * fx
cv2.namedWindow("stereo_pair")
backend_cfg = habitat_sim.SimulatorConfiguration()
backend_cfg.scene.id = ("/home/wbk/Desktop/gibson/Wyatt.glb")
# First, let's create a stereo RGB agent
left_rgb_sensor = habitat_sim.SensorSpec()
# Give it the uuid of left_sensor, this will also be how we
# index the observations to retrieve the rendering from this sensor
left_rgb_sensor.uuid = "left_rgb"
left_rgb_sensor.resolution = [Height, Width]
# The left RGB sensor will be 1.5 meters off the ground
# and 0.25 meters to the left of the center of the agent
left_rgb_sensor.position = Cam_H * habitat_sim.geo.UP + Baseline * habitat_sim.geo.LEFT
right_rgb_sensor = habitat_sim.SensorSpec()
right_rgb_sensor.uuid = "right_rgb"
right_rgb_sensor.resolution = [Height, Width]
right_rgb_sensor.position = Cam_H * habitat_sim.geo.UP + Baseline * habitat_sim.geo.RIGHT
center_rgb_sensor = habitat_sim.SensorSpec()
center_rgb_sensor.uuid = "center_rgb"
def __init__(self,
scene,
result_folder,
depth_camera=False,
loc_depth_cam='c',
foveation=False):
self.agent_config = habitat_sim.AgentConfiguration()
# Left sensor - # oreo perspective - staring at -ive z
self.left_sensor = habitat_sim.SensorSpec()
self.left_sensor.sensor_type = habitat_sim.SensorType.COLOR
self.left_sensor.resolution = sensor_resolution
self.left_sensor.uuid = "left_rgb_sensor"
self.left_sensor.position = [-eye_separation / 2, 0.0, 0.0]
self.left_sensor.orientation = np.array([0.0, 0.0, 0.0], dtype=float)
self.left_sensor_hfov = math.radians(
int(self.left_sensor.parameters['hfov']))
self.focal_distance = sensor_resolution[0] / (
2 * math.tan(self.left_sensor_hfov / 2))
# Right sensor - # oreo perspective - staring at -ive z
self.right_sensor = habitat_sim.SensorSpec()
self.right_sensor.sensor_type = habitat_sim.SensorType.COLOR
self.right_sensor.resolution = sensor_resolution
self.right_sensor.uuid = "right_rgb_sensor"
self.right_sensor.position = [eye_separation / 2, 0.0, 0.0]
self.right_sensor.orientation = np.array([0.0, 0.0, 0.0], dtype=float)
self.right_sensor_hfov = math.radians(
int(self.right_sensor.parameters['hfov']))
if self.right_sensor_hfov != self.left_sensor_hfov:
print("Warning - Right and Left Sensor widths are not identical!")
# Depth camera - At the origin of the reference coordinate axes (habitat frame)
if depth_camera:
self.num_sensors = 3
self.depth_sensor = habitat_sim.SensorSpec()
self.depth_sensor.sensor_type = habitat_sim.SensorType.DEPTH
self.depth_sensor.resolution = sensor_resolution
self.depth_sensor.uuid = "depth_sensor"
if loc_depth_cam == 'l':
self.depth_sensor.position = self.left_sensor.position
elif loc_depth_cam == 'r':
self.depth_sensor.position = self.right_sensor.position
else:
self.depth_sensor.position = [0.0, 0.0, 0.0]
self.depth_sensor.orientation = np.array([0.0, 0.0, 0.0],
dtype=float)
self.agent_config.sensor_specifications = [
self.right_sensor, self.left_sensor, self.depth_sensor
]
else:
self.num_sensors = 2
self.agent_config.sensor_specifications = [
self.right_sensor, self.left_sensor
]
self.backend_cfg = habitat_sim.SimulatorConfiguration()
if foveation:
self.backend_cfg.foveation_distortion = True
self.backend_cfg.scene.id = scene #This works in older habitat versions
# self.backend_cfg.scene_id = scene #newer versions like the colab install
self.destination = os.path.realpath(result_folder)
if not os.path.isdir(self.destination):
os.makedirs(self.destination)
# Tie the backend of the simulator and the list of agent configurations (only one)
self.sim_configuration = habitat_sim.Configuration(
self.backend_cfg, [self.agent_config])
self.sim = habitat_sim.Simulator(self.sim_configuration)
self.agent_id = self.backend_cfg.default_agent_id
self.agent = self.sim.get_agent(self.agent_id)
self.initial_agent_state = self.agent.get_state()
print(
f"Agent rotation {self.initial_agent_state.rotation} Agent position {self.initial_agent_state.position}"
)
# agent_head_neck_rotation (not a part of habitat api to keep track of head/neck rotation wrt to the agent.
# HabitatAI api agent rotation is not rotation of agent wrt to WCS followed by rotation of head/neck
self.agent_head_neck_rotation = np.quaternion(1, 0, 0, 0)
self.counter = 0 # counter for saccade file numbering
self.filename = self.create_unique_filename(scene)
self.my_images = self.get_sensor_observations()
self.current_saved_image = "empty"
return
def make_cfg(self, settings):
sim_cfg = habitat_sim.SimulatorConfiguration()
sim_cfg.gpu_device_id = 0
sim_cfg.scene.id = settings["scene"]
# Note: all sensors must have the same resolution
sensors = {
"color_sensor": {
"sensor_type": habitat_sim.SensorType.COLOR,
"resolution": [settings["height"], settings["width"]],
"position": [0.0, settings["sensor_height"], 0.0],
},
"depth_sensor": {
"sensor_type": habitat_sim.SensorType.DEPTH,
"resolution": [settings["height"], settings["width"]],
"position": [0.0, settings["sensor_height"], 0.0],
},
"semantic_sensor": {
"sensor_type": habitat_sim.SensorType.SEMANTIC,
"resolution": [settings["height"], settings["width"]],
"position": [0.0, settings["sensor_height"], 0.0],
},
}
sensor_specs = []
for sensor_uuid, sensor_params in sensors.items():
# st()
if settings[sensor_uuid]:
sensor_spec = habitat_sim.SensorSpec()
sensor_spec.uuid = sensor_uuid
sensor_spec.sensor_type = sensor_params["sensor_type"]
sensor_spec.resolution = sensor_params["resolution"]
sensor_spec.position = sensor_params["position"]
if sensor_uuid == 'depth_sensor' and self.do_depth_noise:
# add depth noise
sensor_spec.noise_model = "SaltAndPepperNoiseModel"
# sensor_spec.noise_model_kwargs = dict(noise_multiplier=1)
sensor_specs.append(sensor_spec)
# Here you can specify the amount of displacement in a forward action and the turn angle
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
agent_cfg.action_space = {
"do_nothing":
habitat_sim.agent.ActionSpec(
"move_forward", habitat_sim.agent.ActuationSpec(amount=0.)),
"move_forward":
habitat_sim.agent.ActionSpec(
"move_forward", habitat_sim.agent.ActuationSpec(amount=0.25)),
"turn_left":
habitat_sim.agent.ActionSpec(
"turn_left",
habitat_sim.agent.ActuationSpec(amount=self.rot_interval)),
"turn_right":
habitat_sim.agent.ActionSpec(
"turn_right",
habitat_sim.agent.ActuationSpec(amount=self.rot_interval)),
"look_up":
habitat_sim.ActionSpec(
"look_up",
habitat_sim.ActuationSpec(amount=self.rot_interval)),
"look_down":
habitat_sim.ActionSpec(
"look_down",
habitat_sim.ActuationSpec(amount=self.rot_interval)),
"look_down_init":
habitat_sim.ActionSpec("look_down",
habitat_sim.ActuationSpec(amount=100.0))
}
return habitat_sim.Configuration(sim_cfg, [agent_cfg]), sim_cfg
def main(display=True):
global cv2
# Only import cv2 if we are doing to display
if display:
import cv2 as _cv2
cv2 = _cv2
cv2.namedWindow("stereo_pair")
backend_cfg = habitat_sim.SimulatorConfiguration()
backend_cfg.scene.id = (
"data/scene_datasets/habitat-test-scenes/skokloster-castle.glb"
)
# First, let's create a stereo RGB agent
left_rgb_sensor = habitat_sim.SensorSpec()
# Give it the uuid of left_sensor, this will also be how we
# index the observations to retrieve the rendering from this sensor
left_rgb_sensor.uuid = "left_sensor"
left_rgb_sensor.resolution = [512, 512]
# The left RGB sensor will be 1.5 meters off the ground
# and 0.25 meters to the left of the center of the agent
left_rgb_sensor.position = 1.5 * habitat_sim.geo.UP + 0.25 * habitat_sim.geo.LEFT
# Same deal with the right sensor
right_rgb_sensor = habitat_sim.SensorSpec()
right_rgb_sensor.uuid = "right_sensor"
right_rgb_sensor.resolution = [512, 512]
# The right RGB sensor will be 1.5 meters off the ground
# and 0.25 meters to the right of the center of the agent
right_rgb_sensor.position = 1.5 * habitat_sim.geo.UP + 0.25 * habitat_sim.geo.RIGHT
agent_config = habitat_sim.AgentConfiguration()
# Now we simly set the agent's list of sensor specs to be the two specs for our two sensors
agent_config.sensor_specifications = [left_rgb_sensor, right_rgb_sensor]
sim = habitat_sim.Simulator(habitat_sim.Configuration(backend_cfg, [agent_config]))
_render(sim, display)
sim.close()
# Now let's do the exact same thing but for a depth camera stereo pair!
left_depth_sensor = habitat_sim.SensorSpec()
left_depth_sensor.uuid = "left_sensor"
left_depth_sensor.resolution = [512, 512]
left_depth_sensor.position = 1.5 * habitat_sim.geo.UP + 0.25 * habitat_sim.geo.LEFT
# The only difference is that we set the sensor type to DEPTH
left_depth_sensor.sensor_type = habitat_sim.SensorType.DEPTH
right_depth_sensor = habitat_sim.SensorSpec()
right_depth_sensor.uuid = "right_sensor"
right_depth_sensor.resolution = [512, 512]
right_depth_sensor.position = (
1.5 * habitat_sim.geo.UP + 0.25 * habitat_sim.geo.RIGHT
)
# The only difference is that we set the sensor type to DEPTH
right_depth_sensor.sensor_type = habitat_sim.SensorType.DEPTH
agent_config = habitat_sim.AgentConfiguration()
agent_config.sensor_specifications = [left_depth_sensor, right_depth_sensor]
sim = habitat_sim.Simulator(habitat_sim.Configuration(backend_cfg, [agent_config]))
_render(sim, display, depth=True)
