def make_configuration(scene_file):
# simulator configuration
backend_cfg = habitat_sim.SimulatorConfiguration()
backend_cfg.scene_id = scene_file
backend_cfg.enable_physics = True
# sensor configurations
# Note: all sensors must have the same resolution
# setup rgb and semantic sensors
camera_resolution = [1080, 960]
sensor_specs = []
rgba_camera_spec = habitat_sim.CameraSensorSpec()
rgba_camera_spec.uuid = "rgba_camera"
rgba_camera_spec.sensor_type = habitat_sim.SensorType.COLOR
rgba_camera_spec.resolution = camera_resolution
rgba_camera_spec.postition = [0.0, 1.5, 0.0] # ::: fix y to be 0 later
rgba_camera_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(rgba_camera_spec)
semantic_camera_spec = habitat_sim.CameraSensorSpec()
semantic_camera_spec.uuid = "semantic_camera"
semantic_camera_spec.sensor_type = habitat_sim.SensorType.SEMANTIC
semantic_camera_spec.resolution = camera_resolution
semantic_camera_spec.postition = [0.0, 1.5, 0.0]
semantic_camera_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(semantic_camera_spec)
# agent configuration
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
return habitat_sim.Configuration(backend_cfg, [agent_cfg])
def _config_sim(self, scene_filepath, img_size):
settings = {
"width": img_size[1], # Spatial resolution of the observations
"height": img_size[0],
"scene": scene_filepath, # Scene path
"default_agent": 0,
"sensor_height": 1.5, # Height of sensors in meters
"color_sensor": True, # RGBA sensor
"semantic_sensor": True, # Semantic sensor
"depth_sensor": True, # Depth sensor
"silent": True,
}
sim_cfg = hsim.SimulatorConfiguration()
sim_cfg.enable_physics = False
sim_cfg.gpu_device_id = 0
sim_cfg.scene_id = settings["scene"]
# define default sensor parameters (see src/esp/Sensor/Sensor.h)
sensor_specs = []
if settings["color_sensor"]:
color_sensor_spec = habitat_sim.CameraSensorSpec()
color_sensor_spec.uuid = "color_sensor"
color_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
color_sensor_spec.resolution = [
settings["height"], settings["width"]
]
color_sensor_spec.postition = [0.0, settings["sensor_height"], 0.0]
color_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(color_sensor_spec)
if settings["depth_sensor"]:
depth_sensor_spec = habitat_sim.CameraSensorSpec()
depth_sensor_spec.uuid = "depth_sensor"
depth_sensor_spec.sensor_type = habitat_sim.SensorType.DEPTH
depth_sensor_spec.resolution = [
settings["height"], settings["width"]
]
depth_sensor_spec.postition = [0.0, settings["sensor_height"], 0.0]
depth_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(depth_sensor_spec)
if settings["semantic_sensor"]:
semantic_sensor_spec = habitat_sim.CameraSensorSpec()
semantic_sensor_spec.uuid = "semantic_sensor"
semantic_sensor_spec.sensor_type = habitat_sim.SensorType.SEMANTIC
semantic_sensor_spec.resolution = [
settings["height"], settings["width"]
]
semantic_sensor_spec.postition = [
0.0, settings["sensor_height"], 0.0
]
semantic_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(semantic_sensor_spec)
# create agent specifications
agent_cfg = AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
return habitat_sim.Configuration(sim_cfg, [agent_cfg])
def make_cfg(settings):
sim_cfg = habitat_sim.SimulatorConfiguration()
sim_cfg.gpu_device_id = 0
sim_cfg.scene_id = settings["scene"]
sim_cfg.enable_physics = settings["enable_physics"]
# Note: all sensors must have the same resolution
sensor_specs = []
# First, let's create a stereo RGB agent
left_rgb_sensor = habitat_sim.bindings.CameraSensorSpec()
# Give it the uuid of left_sensor, this will also be how we
# index the observations to retrieve the rendering from this sensor
left_rgb_sensor.uuid = "left_sensor"
left_rgb_sensor.resolution = [settings["height"], settings["width"]]
# The left RGB sensor will be 1.5 meters off the ground
# and 0.25 meters to the left of the center of the agent
left_rgb_sensor.position = 1.5 * habitat_sim.geo.UP + 0.05 * habitat_sim.geo.LEFT
sensor_specs.append(left_rgb_sensor)
depth_sensor_spec = habitat_sim.CameraSensorSpec()
depth_sensor_spec.uuid = "depth_sensor"
depth_sensor_spec.sensor_type = habitat_sim.SensorType.DEPTH
depth_sensor_spec.resolution = [settings["height"], settings["width"]]
depth_sensor_spec.postition = 1.5 * habitat_sim.geo.UP + 0.05 * habitat_sim.geo.LEFT
depth_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(depth_sensor_spec)
# Same deal with the right sensor
right_rgb_sensor = habitat_sim.CameraSensorSpec()
right_rgb_sensor.uuid = "right_sensor"
right_rgb_sensor.resolution = [settings["height"], settings["width"]]
# The right RGB sensor will be 1.5 meters off the ground
# and 0.25 meters to the right of the center of the agent
right_rgb_sensor.position = 1.5 * habitat_sim.geo.UP + 0.05 * habitat_sim.geo.RIGHT
sensor_specs.append(right_rgb_sensor)
# Here you can specify the amount of displacement in a forward action and the turn angle
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
agent_cfg.action_space = {
"move_forward":
habitat_sim.agent.ActionSpec(
"move_forward", habitat_sim.agent.ActuationSpec(amount=0.05)),
"turn_left":
habitat_sim.agent.ActionSpec(
"turn_left", habitat_sim.agent.ActuationSpec(amount=1.5)),
"turn_right":
habitat_sim.agent.ActionSpec(
"turn_right", habitat_sim.agent.ActuationSpec(amount=1.5)),
"do_nothing":
habitat_sim.agent.ActionSpec(
"turn_right", habitat_sim.agent.ActuationSpec(amount=0.0))
}
return habitat_sim.Configuration(sim_cfg, [agent_cfg])
def make_cfg(settings):
sim_cfg = habitat_sim.SimulatorConfiguration()
sim_cfg.gpu_device_id = 0
sim_cfg.scene_id = settings["scene"]
sim_cfg.enable_physics = settings["enable_physics"]
# Note: all sensors must have the same resolution
sensor_specs = []
color_sensor_spec = habitat_sim.CameraSensorSpec()
color_sensor_spec.uuid = "color_sensor"
color_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
color_sensor_spec.resolution = [settings["height"], settings["width"]]
color_sensor_spec.postition = [0.0, settings["sensor_height"], 0.0]
color_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(color_sensor_spec)
depth_sensor_spec = habitat_sim.CameraSensorSpec()
depth_sensor_spec.uuid = "depth_sensor"
depth_sensor_spec.sensor_type = habitat_sim.SensorType.DEPTH
depth_sensor_spec.resolution = [settings["height"], settings["width"]]
depth_sensor_spec.postition = [0.0, settings["sensor_height"], 0.0]
depth_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(depth_sensor_spec)
semantic_sensor_spec = habitat_sim.CameraSensorSpec()
semantic_sensor_spec.uuid = "semantic_sensor"
semantic_sensor_spec.sensor_type = habitat_sim.SensorType.SEMANTIC
semantic_sensor_spec.resolution = [settings["height"], settings["width"]]
semantic_sensor_spec.postition = [0.0, settings["sensor_height"], 0.0]
semantic_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(semantic_sensor_spec)
# Here you can specify the amount of displacement in a forward action and the turn angle
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
agent_cfg.action_space = {
"move_forward":
habitat_sim.agent.ActionSpec(
"move_forward", habitat_sim.agent.ActuationSpec(amount=0.03)),
"turn_left":
habitat_sim.agent.ActionSpec(
"turn_left", habitat_sim.agent.ActuationSpec(amount=0.5)),
"turn_right":
habitat_sim.agent.ActionSpec(
"turn_right", habitat_sim.agent.ActuationSpec(amount=0.5)),
"do_nothing":
habitat_sim.agent.ActionSpec(
"turn_right", habitat_sim.agent.ActuationSpec(amount=0.0))
}
return habitat_sim.Configuration(sim_cfg, [agent_cfg])
def make_configuration():
# simulator configuration
backend_cfg = habitat_sim.SimulatorConfiguration()
backend_cfg.scene_id = os.path.join(
data_path, "scene_datasets/habitat-test-scenes/apartment_1.glb")
assert os.path.exists(backend_cfg.scene_id)
backend_cfg.enable_physics = True
# sensor configurations
# Note: all sensors must have the same resolution
# setup 2 rgb sensors for 1st and 3rd person views
camera_resolution = [544, 720]
sensor_specs = []
rgba_camera_1stperson_spec = habitat_sim.CameraSensorSpec()
rgba_camera_1stperson_spec.uuid = "rgba_camera_1stperson"
rgba_camera_1stperson_spec.sensor_type = habitat_sim.SensorType.COLOR
rgba_camera_1stperson_spec.resolution = camera_resolution
rgba_camera_1stperson_spec.postition = [0.0, 0.6, 0.0]
rgba_camera_1stperson_spec.orientation = [0.0, 0.0, 0.0]
rgba_camera_1stperson_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(rgba_camera_1stperson_spec)
depth_camera_1stperson_spec = habitat_sim.CameraSensorSpec()
depth_camera_1stperson_spec.uuid = "depth_camera_1stperson"
depth_camera_1stperson_spec.sensor_type = habitat_sim.SensorType.DEPTH
depth_camera_1stperson_spec.resolution = camera_resolution
depth_camera_1stperson_spec.postition = [0.0, 0.6, 0.0]
depth_camera_1stperson_spec.orientation = [0.0, 0.0, 0.0]
depth_camera_1stperson_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(depth_camera_1stperson_spec)
rgba_camera_3rdperson_spec = habitat_sim.CameraSensorSpec()
rgba_camera_3rdperson_spec.uuid = "rgba_camera_3rdperson"
rgba_camera_3rdperson_spec.sensor_type = habitat_sim.SensorType.COLOR
rgba_camera_3rdperson_spec.resolution = camera_resolution
rgba_camera_3rdperson_spec.postition = [0.0, 1.0, 0.3]
rgba_camera_3rdperson_spec.orientation = [-45, 0.0, 0.0]
rgba_camera_3rdperson_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(rgba_camera_3rdperson_spec)
# agent configuration
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
return habitat_sim.Configuration(backend_cfg, [agent_cfg])
def create_camera_spec(**kw_args):
camera_sensor_spec = habitat_sim.CameraSensorSpec()
camera_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
camera_sensor_spec.resolution = [settings["height"], settings["width"]]
camera_sensor_spec.position = [0, settings["sensor_height"], 0]
for k in kw_args:
setattr(camera_sensor_spec, k, kw_args[k])
return camera_sensor_spec
def make_configuration():
# simulator configuration
backend_cfg = habitat_sim.SimulatorConfiguration()
backend_cfg.scene_id = "data/scene_datasets/habitat-test-scenes/van-gogh-room.glb"
backend_cfg.enable_physics = True
# agent configuration
sensor_cfg = habitat_sim.CameraSensorSpec()
sensor_cfg.resolution = [1080, 960]
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = [sensor_cfg]
return habitat_sim.Configuration(backend_cfg, [agent_cfg])
def make_configuration():
# simulator configuration
backend_cfg = habitat_sim.SimulatorConfiguration()
backend_cfg.scene_id = os.path.join(
data_path, "scene_datasets/habitat-test-scenes/apartment_1.glb")
assert os.path.exists(backend_cfg.scene_id)
backend_cfg.enable_physics = True
# Enable gfx replay save. See also our call to sim.gfx_replay_manager.save_keyframe()
# below.
backend_cfg.enable_gfx_replay_save = True
sensor_cfg = habitat_sim.CameraSensorSpec()
sensor_cfg.resolution = [544, 720]
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = [sensor_cfg]
return habitat_sim.Configuration(backend_cfg, [agent_cfg])
def make_simple_cfg(settings):
# simulator backend
sim_cfg = habitat_sim.SimulatorConfiguration()
sim_cfg.scene_id = settings["scene"]
# agent
agent_cfg = habitat_sim.agent.AgentConfiguration()
# In the 1st example, we attach only one sensor,
# a RGB visual sensor, to the agent
rgb_sensor_spec = habitat_sim.CameraSensorSpec()
rgb_sensor_spec.uuid = "color_sensor"
rgb_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
rgb_sensor_spec.resolution = [settings["height"], settings["width"]]
rgb_sensor_spec.position = [0.0, settings["sensor_height"], 0.0]
agent_cfg.sensor_specifications = [rgb_sensor_spec]
return habitat_sim.Configuration(sim_cfg, [agent_cfg])
def make_configuration():
# simulator configuration
backend_cfg = habitat_sim.SimulatorConfiguration()
backend_cfg.scene_id = "NONE"
backend_cfg.enable_physics = True
# sensor configurations
sensor_specs = []
sensor_spec = habitat_sim.CameraSensorSpec()
sensor_spec.uuid = "rgba_camera_1stperson"
sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
sensor_spec.resolution = [544, 720]
sensor_specs.append(sensor_spec)
# agent configuration
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
return habitat_sim.Configuration(backend_cfg, [agent_cfg])
def make_cfg(settings):
sim_cfg = habitat_sim.SimulatorConfiguration()
sim_cfg.gpu_device_id = 0
sim_cfg.scene_dataset_config_file = settings["scene_dataset"]
sim_cfg.scene_id = settings["scene"]
sim_cfg.enable_physics = settings["enable_physics"]
# Specify the location of the scene dataset
if "scene_dataset_config" in settings:
sim_cfg.scene_dataset_config_file = settings["scene_dataset_config"]
if "override_scene_light_defaults" in settings:
sim_cfg.override_scene_light_defaults = settings[
"override_scene_light_defaults"]
if "scene_light_setup" in settings:
sim_cfg.scene_light_setup = settings["scene_light_setup"]
# Note: all sensors must have the same resolution
sensor_specs = []
color_sensor_1st_person_spec = habitat_sim.CameraSensorSpec()
color_sensor_1st_person_spec.uuid = "color_sensor_1st_person"
color_sensor_1st_person_spec.sensor_type = habitat_sim.SensorType.COLOR
color_sensor_1st_person_spec.resolution = [
settings["height"],
settings["width"],
]
color_sensor_1st_person_spec.postition = [
0.0, settings["sensor_height"], 0.0
]
color_sensor_1st_person_spec.orientation = [
settings["sensor_pitch"],
0.0,
0.0,
]
color_sensor_1st_person_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(color_sensor_1st_person_spec)
# Here you can specify the amount of displacement in a forward action and the turn angle
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
return habitat_sim.Configuration(sim_cfg, [agent_cfg])
def create_sim_config(
self, _sensor_suite: SensorSuite) -> habitat_sim.Configuration:
sim_config = habitat_sim.SimulatorConfiguration()
# Check if Habitat-Sim is post Scene Config Update
if not hasattr(sim_config, "scene_id"):
raise RuntimeError(
"Incompatible version of Habitat-Sim detected, please upgrade habitat_sim"
)
overwrite_config(
config_from=self.habitat_config.HABITAT_SIM_V0,
config_to=sim_config,
# Ignore key as it gets propogated to sensor below
ignore_keys={"gpu_gpu"},
)
sim_config.scene_id = self.habitat_config.SCENE
agent_config = habitat_sim.AgentConfiguration()
overwrite_config(
config_from=self._get_agent_config(),
config_to=agent_config,
# These keys are only used by Hab-Lab
ignore_keys={
"is_set_start_state",
# This is the Sensor Config. Unpacked below
"sensors",
"start_position",
"start_rotation",
},
)
sensor_specifications = []
VisualSensorTypeSet = {
habitat_sim.SensorType.COLOR,
habitat_sim.SensorType.DEPTH,
habitat_sim.SensorType.SEMANTIC,
}
CameraSensorSubTypeSet = {
habitat_sim.SensorSubType.PINHOLE,
habitat_sim.SensorSubType.ORTHOGRAPHIC,
}
for sensor in _sensor_suite.sensors.values():
# Check if type VisualSensorSpec, we know that Sensor is one of HabitatSimRGBSensor, HabitatSimDepthSensor, HabitatSimSemanticSensor
if (getattr(sensor, "sim_sensor_type", [])
not in VisualSensorTypeSet):
raise ValueError(
f"""{getattr(sensor, "sim_sensor_type", [])} is an illegal sensorType that is not implemented yet"""
)
# Check if type CameraSensorSpec
if (getattr(sensor, "sim_sensor_subtype", [])
not in CameraSensorSubTypeSet):
raise ValueError(
f"""{getattr(sensor, "sim_sensor_subtype", [])} is an illegal sensorSubType for a VisualSensor"""
)
# TODO: Implement checks for other types of SensorSpecs
sim_sensor_cfg = habitat_sim.CameraSensorSpec()
# TODO Handle configs for custom VisualSensors that might need
# their own ignore_keys. Maybe with special key / checking
# SensorType
overwrite_config(
config_from=sensor.config,
config_to=sim_sensor_cfg,
# These keys are only used by Hab-Lab
# or translated into the sensor config manually
ignore_keys={
"height",
"hfov",
"max_depth",
"min_depth",
"normalize_depth",
"type",
"width",
},
)
sim_sensor_cfg.uuid = sensor.uuid
sim_sensor_cfg.resolution = list(
sensor.observation_space.shape[:2])
# TODO(maksymets): Add configure method to Sensor API to avoid
# accessing child attributes through parent interface
# We know that the Sensor has to be one of these Sensors
sensor = cast(HabitatSimVizSensors, sensor)
sim_sensor_cfg.sensor_type = sensor.sim_sensor_type
sim_sensor_cfg.sensor_subtype = sensor.sim_sensor_subtype
sim_sensor_cfg.gpu2gpu_transfer = (
self.habitat_config.HABITAT_SIM_V0.GPU_GPU)
sensor_specifications.append(sim_sensor_cfg)
agent_config.sensor_specifications = sensor_specifications
agent_config.action_space = registry.get_action_space_configuration(
self.habitat_config.ACTION_SPACE_CONFIG)(
self.habitat_config).get()
return habitat_sim.Configuration(sim_config, [agent_config])
def main():
backend_cfg = habitat_sim.SimulatorConfiguration()
backend_cfg.scene_id = (
"data/scene_datasets/habitat-test-scenes/skokloster-castle.glb")
backend_cfg.enable_physics = True
# Leaving the context with the background thread can improve
# performance, but makes it so that sim.get_sensor_observations_async_finish()
# no longer returns the context back to the main thread. If you set this to
# true and get context assert failures, call sim.renderer.acquire_gl_context()
# to move the OpenGL context back to the main thread.
# The default is False
backend_cfg.leave_context_with_background_renderer = False
agent_cfg = habitat_sim.AgentConfiguration(
sensor_specifications=[habitat_sim.CameraSensorSpec()])
cfg = habitat_sim.Configuration(backend_cfg, [agent_cfg])
# Number of physics steps per render
n_physics_steps_per_render = 4
# Render at 30 hz
physics_step_time = 1.0 / (30 * n_physics_steps_per_render)
sim = habitat_sim.Simulator(cfg)
for i in range(n_physics_steps_per_render):
# Always call sim.step_physics when not using async rendering
if i == 0:
sim.start_async_render_and_step_physics(physics_step_time)
else:
sim.step_physics(physics_step_time)
# Call sim.renderer.acquire_gl_context() if you need to create or destroy an object.
# Note that this will block until rendering is done.
obs = sim.get_sensor_observations_async_finish()
# To change back to normal rendering, use
# obs = sim.get_sensor_observations()
# You can add/remove objects after the call to get_sensor_observations_async_finish()
# if backend_cfg.leave_context_with_background_renderer was left as False, it that as
# true, you'd need to call
# sim.renderer.acquire_gl_context()
# Calling acquire_gl_context() is a noop if the main thread already has the OpenGL context
backend_cfg = habitat_sim.SimulatorConfiguration()
backend_cfg.scene_id = "data/scene_datasets/habitat-test-scenes/apartment_1.glb"
backend_cfg.enable_physics = True
new_cfg = habitat_sim.Configuration(backend_cfg, [agent_cfg])
# To create a new instance of the simulator to swap scene or similar,
# do not delete the simulator instance as this may leak GPU memory
# Instead, call sim.close(destroy=False) as this will close/delete
# everything except the OpenGL context and the background render
# thread.
sim.close(destroy=False)
# Then use reconfigure.
sim.reconfigure(new_cfg)
# If
# sim.close()
# sim = habitat_sim.Simulator(new_cfg)
# was used, that could end up leaking memory
sim.start_async_render_and_step_physics(physics_step_time)
obs = sim.get_sensor_observations_async_finish() # noqa: F841
# Call close with destroy=True here because this example is over :)
sim.close(destroy=True)
def make_configuration(
habitat_dir: str,
glb_path: str,
) -> habitat_sim.SimulatorConfiguration:
"""Create a habitat_sim.SimulatorConfiguration object, and populate it
with the scene from the glb_path.
Args:
habitat_dir: the directory containing habitat-sim. Mainly used as root
of glb_path (if it's not an absolute path) and the physics config
file (typically `data/default.physics_config.json`).
glb_path: path to the .glb file. If a relative path, assumed to be
relative to `habitat_dir`.
Returns:
habitat_sim.SimulatorConfiguration object with reasonable values and
loaded with glb scene.
"""
if not os.path.isabs(glb_path):
glb_path = os.path.join(habitat_dir, glb_path)
# simulator configuration
backend_cfg = habitat_sim.SimulatorConfiguration()
backend_cfg.scene_id = glb_path
backend_cfg.enable_physics = True
backend_cfg.physics_config_file = os.path.join(
habitat_dir, backend_cfg.physics_config_file
)
# sensor configurations
# Note: all sensors must have the same resolution
# setup 2 rgb sensors for 1st and 3rd person views
camera_resolution = [540, 720]
sensors = {
"rgba_camera_1stperson": {
"sensor_type": habitat_sim.SensorType.COLOR,
"resolution": camera_resolution,
"position": [0.0, 0.0, 0.0],
"orientation": [0.0, 0.0, 0.0],
},
"depth_camera_1stperson": {
"sensor_type": habitat_sim.SensorType.DEPTH,
"resolution": camera_resolution,
"position": [0.0, 0.0, 0.0],
"orientation": [0.0, 0.0, 0.0],
},
}
sensor_specs = []
for sensor_uuid, sensor_params in sensors.items():
sensor_spec = habitat_sim.CameraSensorSpec()
sensor_spec.uuid = sensor_uuid
sensor_spec.sensor_type = sensor_params["sensor_type"]
sensor_spec.resolution = sensor_params["resolution"]
sensor_spec.position = sensor_params["position"]
sensor_spec.orientation = sensor_params["orientation"]
sensor_specs.append(sensor_spec)
# agent configuration
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
return habitat_sim.Configuration(backend_cfg, [agent_cfg])
def make_cfg(settings):
sim_cfg = habitat_sim.SimulatorConfiguration()
if "frustum_culling" in settings:
sim_cfg.frustum_culling = settings["frustum_culling"]
else:
sim_cfg.frustum_culling = False
if "enable_physics" in settings:
sim_cfg.enable_physics = settings["enable_physics"]
if "physics_config_file" in settings:
sim_cfg.physics_config_file = settings["physics_config_file"]
if not settings["silent"]:
print("sim_cfg.physics_config_file = " + sim_cfg.physics_config_file)
if "scene_light_setup" in settings:
sim_cfg.scene_light_setup = settings["scene_light_setup"]
sim_cfg.gpu_device_id = 0
if not hasattr(sim_cfg, "scene_id"):
raise RuntimeError(
"Error: Please upgrade habitat-sim. SimulatorConfig API version mismatch"
)
sim_cfg.scene_id = settings["scene"]
# define default sensor parameters (see src/esp/Sensor/Sensor.h)
sensor_specs = []
if settings["color_sensor"]:
color_sensor_spec = habitat_sim.CameraSensorSpec()
color_sensor_spec.uuid = "color_sensor"
color_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
color_sensor_spec.resolution = [settings["height"], settings["width"]]
color_sensor_spec.position = [0, settings["sensor_height"], 0]
color_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(color_sensor_spec)
if settings["depth_sensor"]:
depth_sensor_spec = habitat_sim.CameraSensorSpec()
depth_sensor_spec.uuid = "depth_sensor"
depth_sensor_spec.sensor_type = habitat_sim.SensorType.DEPTH
depth_sensor_spec.resolution = [settings["height"], settings["width"]]
depth_sensor_spec.position = [0, settings["sensor_height"], 0]
depth_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(depth_sensor_spec)
if settings["semantic_sensor"]:
semantic_sensor_spec = habitat_sim.CameraSensorSpec()
semantic_sensor_spec.uuid = "semantic_sensor"
semantic_sensor_spec.sensor_type = habitat_sim.SensorType.SEMANTIC
semantic_sensor_spec.resolution = [
settings["height"], settings["width"]
]
semantic_sensor_spec.position = [0, settings["sensor_height"], 0]
semantic_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(semantic_sensor_spec)
if settings["ortho_sensor"]:
ortho_sensor_spec = habitat_sim.CameraSensorSpec()
ortho_sensor_spec.uuid = "ortho_sensor"
ortho_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
ortho_sensor_spec.resolution = [settings["height"], settings["width"]]
ortho_sensor_spec.position = [0, settings["sensor_height"], 0]
ortho_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.ORTHOGRAPHIC
sensor_specs.append(ortho_sensor_spec)
# create agent specifications
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
agent_cfg.action_space = {
"move_forward":
habitat_sim.agent.ActionSpec(
"move_forward", habitat_sim.agent.ActuationSpec(amount=0.25)),
"turn_left":
habitat_sim.agent.ActionSpec(
"turn_left", habitat_sim.agent.ActuationSpec(amount=10.0)),
"turn_right":
habitat_sim.agent.ActionSpec(
"turn_right", habitat_sim.agent.ActuationSpec(amount=10.0)),
}
# override action space to no-op to test physics
if sim_cfg.enable_physics:
agent_cfg.action_space = {
"move_forward":
habitat_sim.agent.ActionSpec(
"move_forward", habitat_sim.agent.ActuationSpec(amount=0.0))
}
return habitat_sim.Configuration(sim_cfg, [agent_cfg])
def make_cfg(settings):
sim_cfg = habitat_sim.SimulatorConfiguration()
if "frustum_culling" in settings:
sim_cfg.frustum_culling = settings["frustum_culling"]
else:
sim_cfg.frustum_culling = False
if "enable_physics" in settings:
sim_cfg.enable_physics = settings["enable_physics"]
if "physics_config_file" in settings:
sim_cfg.physics_config_file = settings["physics_config_file"]
if not settings["silent"]:
print("sim_cfg.physics_config_file = " + sim_cfg.physics_config_file)
if "scene_light_setup" in settings:
sim_cfg.scene_light_setup = settings["scene_light_setup"]
sim_cfg.gpu_device_id = 0
if not hasattr(sim_cfg, "scene_id"):
raise RuntimeError(
"Error: Please upgrade habitat-sim. SimulatorConfig API version mismatch"
)
sim_cfg.scene_id = settings["scene"]
# define default sensor parameters (see src/esp/Sensor/Sensor.h)
sensor_specs = []
if settings["color_sensor"]:
color_sensor_spec = habitat_sim.CameraSensorSpec()
color_sensor_spec.uuid = "color_sensor"
color_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
color_sensor_spec.resolution = [settings["height"], settings["width"]]
color_sensor_spec.position = [0, settings["sensor_height"], 0]
color_sensor_spec.hfov = settings["hfov"]
color_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(color_sensor_spec)
if settings["depth_sensor"]:
depth_sensor_spec = habitat_sim.CameraSensorSpec()
depth_sensor_spec.uuid = "depth_sensor"
depth_sensor_spec.sensor_type = habitat_sim.SensorType.DEPTH
depth_sensor_spec.resolution = [settings["height"], settings["width"]]
depth_sensor_spec.position = [0, settings["sensor_height"], 0]
depth_sensor_spec.hfov = settings["hfov"]
depth_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(depth_sensor_spec)
if settings["semantic_sensor"]:
semantic_sensor_spec = habitat_sim.CameraSensorSpec()
semantic_sensor_spec.uuid = "semantic_sensor"
semantic_sensor_spec.sensor_type = habitat_sim.SensorType.SEMANTIC
semantic_sensor_spec.resolution = [
settings["height"], settings["width"]
]
semantic_sensor_spec.position = [0, settings["sensor_height"], 0]
semantic_sensor_spec.hfov = settings["hfov"]
semantic_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(semantic_sensor_spec)
if settings["ortho_sensor"]:
ortho_sensor_spec = habitat_sim.CameraSensorSpec()
ortho_sensor_spec.uuid = "ortho_sensor"
ortho_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
ortho_sensor_spec.resolution = [settings["height"], settings["width"]]
ortho_sensor_spec.position = [0, settings["sensor_height"], 0]
ortho_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.ORTHOGRAPHIC
sensor_specs.append(ortho_sensor_spec)
# TODO Figure out how to implement copying of specs
def create_fisheye_spec(**kw_args):
fisheye_sensor_spec = habitat_sim.FisheyeSensorDoubleSphereSpec()
fisheye_sensor_spec.uuid = "fisheye_sensor"
fisheye_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
fisheye_sensor_spec.sensor_model_type = (
habitat_sim.FisheyeSensorModelType.DOUBLE_SPHERE)
# The default value (alpha, xi) is set to match the lens "BF2M2020S23" found in Table 3 of this paper:
# Vladyslav Usenko, Nikolaus Demmel and Daniel Cremers: The Double Sphere
# Camera Model, The International Conference on 3D Vision (3DV), 2018
# You can find the intrinsic parameters for the other lenses in the same table as well.
# fisheye_sensor_spec.alpha = 0.59 # default
# fisheye_sensor_spec.xi = -0.18 # default
fisheye_sensor_spec.resolution = [
settings["height"], settings["width"]
]
fisheye_sensor_spec.focal_length = [
min(settings["height"], settings["width"]) * 0.5
] * 2
# The default principal_point_offset is the middle of the image
fisheye_sensor_spec.principal_point_offset = mn.Vector2(
settings["height"] // 2,
settings["width"] // 2,
)
fisheye_sensor_spec.position = [0, settings["sensor_height"], 0]
for k in kw_args:
setattr(fisheye_sensor_spec, k, kw_args[k])
return fisheye_sensor_spec
if settings["fisheye_rgb_sensor"]:
fisheye_rgb_sensor_spec = create_fisheye_spec(
uuid="fisheye_rgb_sensor")
sensor_specs.append(fisheye_rgb_sensor_spec)
if settings["fisheye_depth_sensor"]:
fisheye_depth_sensor_spec = create_fisheye_spec(
uuid="fisheye_depth_sensor",
sensor_type=habitat_sim.SensorType.DEPTH)
sensor_specs.append(fisheye_depth_sensor_spec)
if settings["equirect_rgb_sensor"]:
equirect_rgb_sensor_spec = habitat_sim.EquirectangularSensorSpec()
equirect_rgb_sensor_spec.uuid = "equirect_rgb_sensor"
equirect_rgb_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
equirect_rgb_sensor_spec.resolution = [
settings["height"], settings["width"]
]
equirect_rgb_sensor_spec.position = [0, settings["sensor_height"], 0]
sensor_specs.append(equirect_rgb_sensor_spec)
if settings["equirect_depth_sensor"]:
equirect_depth_sensor_spec = habitat_sim.EquirectangularSensorSpec()
equirect_depth_sensor_spec.uuid = "equirect_depth_sensor"
equirect_depth_sensor_spec.sensor_type = habitat_sim.SensorType.DEPTH
equirect_depth_sensor_spec.resolution = [
settings["height"], settings["width"]
]
equirect_depth_sensor_spec.position = [0, settings["sensor_height"], 0]
sensor_specs.append(equirect_depth_sensor_spec)
# create agent specifications
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
agent_cfg.action_space = {
"move_forward":
habitat_sim.agent.ActionSpec(
"move_forward", habitat_sim.agent.ActuationSpec(amount=0.25)),
"turn_left":
habitat_sim.agent.ActionSpec(
"turn_left", habitat_sim.agent.ActuationSpec(amount=10.0)),
"turn_right":
habitat_sim.agent.ActionSpec(
"turn_right", habitat_sim.agent.ActuationSpec(amount=10.0)),
}
# override action space to no-op to test physics
if sim_cfg.enable_physics:
agent_cfg.action_space = {
"move_forward":
habitat_sim.agent.ActionSpec(
"move_forward", habitat_sim.agent.ActuationSpec(amount=0.0))
}
return habitat_sim.Configuration(sim_cfg, [agent_cfg])
def make_cfg(settings):
sim_cfg = habitat_sim.SimulatorConfiguration()
sim_cfg.gpu_device_id = 0
sim_cfg.scene_id = settings["scene"]
sim_cfg.enable_physics = settings["enable_physics"]
# Note: all sensors must have the same resolution
sensor_specs = []
if settings["color_sensor_1st_person"]:
color_sensor_1st_person_spec = habitat_sim.CameraSensorSpec()
color_sensor_1st_person_spec.uuid = "color_sensor_1st_person"
color_sensor_1st_person_spec.sensor_type = habitat_sim.SensorType.COLOR
color_sensor_1st_person_spec.resolution = [
settings["height"],
settings["width"],
]
color_sensor_1st_person_spec.postition = [0.0, settings["sensor_height"], 0.0]
color_sensor_1st_person_spec.orientation = [
settings["sensor_pitch"],
0.0,
0.0,
]
color_sensor_1st_person_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(color_sensor_1st_person_spec)
if settings["depth_sensor_1st_person"]:
depth_sensor_1st_person_spec = habitat_sim.CameraSensorSpec()
depth_sensor_1st_person_spec.uuid = "depth_sensor_1st_person"
depth_sensor_1st_person_spec.sensor_type = habitat_sim.SensorType.DEPTH
depth_sensor_1st_person_spec.resolution = [
settings["height"],
settings["width"],
]
depth_sensor_1st_person_spec.postition = [0.0, settings["sensor_height"], 0.0]
depth_sensor_1st_person_spec.orientation = [
settings["sensor_pitch"],
0.0,
0.0,
]
depth_sensor_1st_person_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(depth_sensor_1st_person_spec)
if settings["semantic_sensor_1st_person"]:
semantic_sensor_1st_person_spec = habitat_sim.CameraSensorSpec()
semantic_sensor_1st_person_spec.uuid = "semantic_sensor_1st_person"
semantic_sensor_1st_person_spec.sensor_type = habitat_sim.SensorType.SEMANTIC
semantic_sensor_1st_person_spec.resolution = [
settings["height"],
settings["width"],
]
semantic_sensor_1st_person_spec.postition = [
0.0,
settings["sensor_height"],
0.0,
]
semantic_sensor_1st_person_spec.orientation = [
settings["sensor_pitch"],
0.0,
0.0,
]
semantic_sensor_1st_person_spec.sensor_subtype = (
habitat_sim.SensorSubType.PINHOLE
)
sensor_specs.append(semantic_sensor_1st_person_spec)
if settings["color_sensor_3rd_person"]:
color_sensor_3rd_person_spec = habitat_sim.CameraSensorSpec()
color_sensor_3rd_person_spec.uuid = "color_sensor_3rd_person"
color_sensor_3rd_person_spec.sensor_type = habitat_sim.SensorType.COLOR
color_sensor_3rd_person_spec.resolution = [
settings["height"],
settings["width"],
]
color_sensor_3rd_person_spec.postition = [
0.0,
settings["sensor_height"] + 0.2,
0.2,
]
color_sensor_3rd_person_spec.orientation = [-math.pi / 4, 0, 0]
color_sensor_3rd_person_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
sensor_specs.append(color_sensor_3rd_person_spec)
# Here you can specify the amount of displacement in a forward action and the turn angle
agent_cfg = habitat_sim.agent.AgentConfiguration()
agent_cfg.sensor_specifications = sensor_specs
return habitat_sim.Configuration(sim_cfg, [agent_cfg])
def main(display=True):
global cv2
# Only import cv2 if we are doing to display
if display:
import cv2 as _cv2
cv2 = _cv2
cv2.namedWindow("stereo_pair")
backend_cfg = habitat_sim.SimulatorConfiguration()
backend_cfg.scene_id = (
"data/scene_datasets/habitat-test-scenes/skokloster-castle.glb")
# First, let's create a stereo RGB agent
left_rgb_sensor = habitat_sim.bindings.CameraSensorSpec()
# Give it the uuid of left_sensor, this will also be how we
# index the observations to retrieve the rendering from this sensor
left_rgb_sensor.uuid = "left_sensor"
left_rgb_sensor.resolution = [512, 512]
# The left RGB sensor will be 1.5 meters off the ground
# and 0.25 meters to the left of the center of the agent
left_rgb_sensor.position = 1.5 * habitat_sim.geo.UP + 0.25 * habitat_sim.geo.LEFT
# Same deal with the right sensor
right_rgb_sensor = habitat_sim.CameraSensorSpec()
right_rgb_sensor.uuid = "right_sensor"
right_rgb_sensor.resolution = [512, 512]
# The right RGB sensor will be 1.5 meters off the ground
# and 0.25 meters to the right of the center of the agent
right_rgb_sensor.position = 1.5 * habitat_sim.geo.UP + 0.25 * habitat_sim.geo.RIGHT
agent_config = habitat_sim.AgentConfiguration()
# Now we simly set the agent's list of sensor specs to be the two specs for our two sensors
agent_config.sensor_specifications = [left_rgb_sensor, right_rgb_sensor]
sim = habitat_sim.Simulator(
habitat_sim.Configuration(backend_cfg, [agent_config]))
_render(sim, display)
sim.close()
# Now let's do the exact same thing but for a depth camera stereo pair!
left_depth_sensor = habitat_sim.CameraSensorSpec()
left_depth_sensor.uuid = "left_sensor"
left_depth_sensor.resolution = [512, 512]
left_depth_sensor.position = 1.5 * habitat_sim.geo.UP + 0.25 * habitat_sim.geo.LEFT
# The only difference is that we set the sensor type to DEPTH
left_depth_sensor.sensor_type = habitat_sim.SensorType.DEPTH
right_depth_sensor = habitat_sim.CameraSensorSpec()
right_depth_sensor.uuid = "right_sensor"
right_depth_sensor.resolution = [512, 512]
right_depth_sensor.position = (1.5 * habitat_sim.geo.UP +
0.25 * habitat_sim.geo.RIGHT)
# The only difference is that we set the sensor type to DEPTH
right_depth_sensor.sensor_type = habitat_sim.SensorType.DEPTH
agent_config = habitat_sim.AgentConfiguration()
agent_config.sensor_specifications = [
left_depth_sensor, right_depth_sensor
]
sim = habitat_sim.Simulator(
habitat_sim.Configuration(backend_cfg, [agent_config]))
_render(sim, display, depth=True)
