def agent_demo():
config = get_habitat_config(
os.path.join(HABITAT_CONFIGS_DIR, "tasks/pointnav.yaml"))
config.defrost()
config.DATASET.DATA_PATH = os.path.join(
HABITAT_DATASETS_DIR, "pointnav/gibson/v1/train/train.json.gz")
config.DATASET.SCENES_DIR = HABITAT_SCENE_DATASETS_DIR
config.SIMULATOR.HABITAT_SIM_V0.GPU_DEVICE_ID = 0
config.SIMULATOR.TURN_ANGLE = 45
config.freeze()
env = habitat.Env(config=config)
print("Environment creation successful")
observations = env.reset()
cv2.imshow("RGB", transform_rgb_bgr(observations["rgb"]))
print("Agent stepping around inside environment.")
count_steps = 0
action = None
while not env.episode_over:
keystroke = cv2.waitKey(0)
if keystroke == ord(FORWARD_KEY):
action = 1
print("action: FORWARD")
elif keystroke == ord(LEFT_KEY):
action = 2
print("action: LEFT")
elif keystroke == ord(RIGHT_KEY):
action = 3
print("action: RIGHT")
elif keystroke == ord(FINISH):
action = 0
print("action: FINISH")
else:
print("INVALID KEY")
continue
observations = env.step(action)
count_steps += 1
print("Position:", env.sim.get_agent_state().position)
print("Quaternions:", env.sim.get_agent_state().rotation)
quat = Quaternion(env.sim.get_agent_state().rotation.components)
print(quat.radians)
cv2.imshow("RGB", transform_rgb_bgr(observations["rgb"]))
print("Episode finished after {} steps.".format(count_steps))
if action == habitat.SimulatorActions.STOP and observations["pointgoal"][
0] < 0.2:
print("you successfully navigated to destination point")
else:
print("your navigation was unsuccessful")
def create_pointnav_config(
config_yaml_path: str,
mode: str,
scenes_path: str,
simulator_gpu_ids: Sequence[int],
distance_to_goal: float,
rotation_degrees: float,
step_size: float,
max_steps: int,
num_processes: int,
camera_width: int,
camera_height: int,
using_rgb: bool,
using_depth: bool,
) -> habitat.Config:
config = get_habitat_config(config_yaml_path)
config.defrost()
config.NUM_PROCESSES = num_processes
config.SIMULATOR_GPU_IDS = simulator_gpu_ids
config.DATASET.SCENES_DIR = HABITAT_SCENE_DATASETS_DIR
config.DATASET.DATA_PATH = scenes_path
config.SIMULATOR.AGENT_0.SENSORS = []
if using_rgb:
config.SIMULATOR.AGENT_0.SENSORS.append("RGB_SENSOR")
if using_depth:
config.SIMULATOR.AGENT_0.SENSORS.append("DEPTH_SENSOR")
config.SIMULATOR.RGB_SENSOR.WIDTH = camera_width
config.SIMULATOR.RGB_SENSOR.HEIGHT = camera_height
config.SIMULATOR.DEPTH_SENSOR.WIDTH = camera_width
config.SIMULATOR.DEPTH_SENSOR.HEIGHT = camera_height
config.SIMULATOR.TURN_ANGLE = rotation_degrees
config.SIMULATOR.FORWARD_STEP_SIZE = step_size
config.ENVIRONMENT.MAX_EPISODE_STEPS = max_steps
config.TASK.TYPE = "Nav-v0"
config.TASK.SUCCESS_DISTANCE = distance_to_goal
config.TASK.SENSORS = ["POINTGOAL_WITH_GPS_COMPASS_SENSOR"]
config.TASK.POINTGOAL_WITH_GPS_COMPASS_SENSOR.GOAL_FORMAT = "POLAR"
config.TASK.POINTGOAL_WITH_GPS_COMPASS_SENSOR.DIMENSIONALITY = 2
config.TASK.GOAL_SENSOR_UUID = "pointgoal_with_gps_compass"
config.TASK.MEASUREMENTS = ["DISTANCE_TO_GOAL", "SUCCESS", "SPL"]
config.TASK.SPL.TYPE = "SPL"
config.TASK.SPL.SUCCESS_DISTANCE = distance_to_goal
config.TASK.SUCCESS.SUCCESS_DISTANCE = distance_to_goal
config.MODE = mode
config.freeze()
return config
def generate_maps():
config = get_habitat_config(
os.path.join(HABITAT_CONFIGS_DIR, "tasks/pointnav.yaml")
)
config.defrost()
config.DATASET.DATA_PATH = os.path.join(
HABITAT_DATASETS_DIR, "pointnav/gibson/v1/train/train.json.gz"
)
config.DATASET.SCENES_DIR = HABITAT_SCENE_DATASETS_DIR
config.SIMULATOR.HABITAT_SIM_V0.GPU_DEVICE_ID = 0
config.freeze()
dataset = habitat.make_dataset(config.DATASET.TYPE)
scenes = dataset.get_scenes_to_load(config.DATASET)
for scene in scenes:
print("Making environment for:", scene)
config.defrost()
config.DATASET.CONTENT_SCENES = [scene]
config.freeze()
env = habitat.Env(config=config)
make_map(env, scene)
env.close()
class PointNavHabitatRGBPPOTutorialExperimentConfig(ExperimentConfig):
"""A Point Navigation experiment configuration in Habitat."""
# Task Parameters
MAX_STEPS = 500
REWARD_CONFIG = {
"step_penalty": -0.01,
"goal_success_reward": 10.0,
"failed_stop_reward": 0.0,
"shaping_weight": 1.0,
}
DISTANCE_TO_GOAL = 0.2
# Simulator Parameters
CAMERA_WIDTH = 640
CAMERA_HEIGHT = 480
SCREEN_SIZE = 224
# Training Engine Parameters
ADVANCE_SCENE_ROLLOUT_PERIOD: Optional[int] = None
NUM_PROCESSES = max(5 * torch.cuda.device_count() - 1, 4)
TRAINING_GPUS = list(range(torch.cuda.device_count()))
VALIDATION_GPUS = [torch.cuda.device_count() - 1]
TESTING_GPUS = [torch.cuda.device_count() - 1]
task_data_dir_template = os.path.join(HABITAT_DATASETS_DIR,
"pointnav/gibson/v1/{}/{}.json.gz")
TRAIN_SCENES = task_data_dir_template.format(*(["train"] * 2))
VALID_SCENES = task_data_dir_template.format(*(["val"] * 2))
TEST_SCENES = task_data_dir_template.format(*(["test"] * 2))
CONFIG = get_habitat_config(
os.path.join(HABITAT_CONFIGS_DIR, "tasks/pointnav_gibson.yaml"))
CONFIG.defrost()
CONFIG.NUM_PROCESSES = NUM_PROCESSES
CONFIG.SIMULATOR_GPU_IDS = TRAINING_GPUS
CONFIG.DATASET.SCENES_DIR = "habitat/habitat-api/data/scene_datasets/"
CONFIG.DATASET.POINTNAVV1.CONTENT_SCENES = ["*"]
CONFIG.DATASET.DATA_PATH = TRAIN_SCENES
CONFIG.SIMULATOR.AGENT_0.SENSORS = ["RGB_SENSOR"]
CONFIG.SIMULATOR.RGB_SENSOR.WIDTH = CAMERA_WIDTH
CONFIG.SIMULATOR.RGB_SENSOR.HEIGHT = CAMERA_HEIGHT
CONFIG.SIMULATOR.TURN_ANGLE = 30
CONFIG.SIMULATOR.FORWARD_STEP_SIZE = 0.25
CONFIG.ENVIRONMENT.MAX_EPISODE_STEPS = MAX_STEPS
CONFIG.TASK.TYPE = "Nav-v0"
CONFIG.TASK.SUCCESS_DISTANCE = DISTANCE_TO_GOAL
CONFIG.TASK.SENSORS = ["POINTGOAL_WITH_GPS_COMPASS_SENSOR"]
CONFIG.TASK.POINTGOAL_WITH_GPS_COMPASS_SENSOR.GOAL_FORMAT = "POLAR"
CONFIG.TASK.POINTGOAL_WITH_GPS_COMPASS_SENSOR.DIMENSIONALITY = 2
CONFIG.TASK.GOAL_SENSOR_UUID = "pointgoal_with_gps_compass"
CONFIG.TASK.MEASUREMENTS = ["DISTANCE_TO_GOAL", "SUCCESS", "SPL"]
CONFIG.TASK.SPL.TYPE = "SPL"
CONFIG.TASK.SPL.SUCCESS_DISTANCE = DISTANCE_TO_GOAL
CONFIG.TASK.SUCCESS.SUCCESS_DISTANCE = DISTANCE_TO_GOAL
CONFIG.MODE = "train"
SENSORS = [
RGBSensorHabitat(
height=SCREEN_SIZE,
width=SCREEN_SIZE,
use_resnet_normalization=True,
),
TargetCoordinatesSensorHabitat(coordinate_dims=2),
]
PREPROCESSORS = [
Builder(
ResNetPreprocessor,
{
"input_height": SCREEN_SIZE,
"input_width": SCREEN_SIZE,
"output_width": 7,
"output_height": 7,
"output_dims": 512,
"pool": False,
"torchvision_resnet_model": models.resnet18,
"input_uuids": ["rgb_lowres"],
"output_uuid": "rgb_resnet",
},
),
]
OBSERVATIONS = [
"rgb_resnet",
"target_coordinates_ind",
]
TRAIN_CONFIGS = construct_env_configs(CONFIG)
@classmethod
def tag(cls):
return "PointNavHabitatRGBPPO"
@classmethod
def training_pipeline(cls, **kwargs):
ppo_steps = int(250000000)
lr = 3e-4
num_mini_batch = 1
update_repeats = 3
num_steps = 30
save_interval = 5000000
log_interval = 10000
gamma = 0.99
use_gae = True
gae_lambda = 0.95
max_grad_norm = 0.5
return TrainingPipeline(
save_interval=save_interval,
metric_accumulate_interval=log_interval,
optimizer_builder=Builder(optim.Adam, dict(lr=lr)),
num_mini_batch=num_mini_batch,
update_repeats=update_repeats,
max_grad_norm=max_grad_norm,
num_steps=num_steps,
named_losses={"ppo_loss": PPO(**PPOConfig)},
gamma=gamma,
use_gae=use_gae,
gae_lambda=gae_lambda,
advance_scene_rollout_period=cls.ADVANCE_SCENE_ROLLOUT_PERIOD,
pipeline_stages=[
PipelineStage(loss_names=["ppo_loss"],
max_stage_steps=ppo_steps)
],
lr_scheduler_builder=Builder(
LambdaLR, {"lr_lambda": LinearDecay(steps=ppo_steps)}),
)
def machine_params(self, mode="train", **kwargs):
if mode == "train":
workers_per_device = 1
gpu_ids = ([] if not torch.cuda.is_available() else
self.TRAINING_GPUS * workers_per_device)
nprocesses = (1 if not torch.cuda.is_available() else
evenly_distribute_count_into_bins(
self.NUM_PROCESSES, len(gpu_ids)))
elif mode == "valid":
nprocesses = 1
gpu_ids = [] if not torch.cuda.is_available(
) else self.VALIDATION_GPUS
elif mode == "test":
nprocesses = 1
gpu_ids = [] if not torch.cuda.is_available(
) else self.TESTING_GPUS
else:
raise NotImplementedError(
"mode must be 'train', 'valid', or 'test'.")
sensor_preprocessor_graph = (SensorPreprocessorGraph(
source_observation_spaces=SensorSuite(
self.SENSORS).observation_spaces,
preprocessors=self.PREPROCESSORS,
) if mode == "train" or (
(isinstance(nprocesses, int) and nprocesses > 0) or
(isinstance(nprocesses, Sequence) and sum(nprocesses) > 0)) else
None)
return MachineParams(
nprocesses=nprocesses,
devices=gpu_ids,
sensor_preprocessor_graph=sensor_preprocessor_graph,
)
# Define Model
@classmethod
def create_model(cls, **kwargs) -> nn.Module:
return ResnetTensorPointNavActorCritic(
action_space=gym.spaces.Discrete(
len(PointNavTask.class_action_names())),
observation_space=kwargs["sensor_preprocessor_graph"].
observation_spaces,
goal_sensor_uuid="target_coordinates_ind",
rgb_resnet_preprocessor_uuid="rgb_resnet",
hidden_size=512,
goal_dims=32,
)
# Define Task Sampler
@classmethod
def make_sampler_fn(cls, **kwargs) -> TaskSampler:
return PointNavTaskSampler(**kwargs)
def train_task_sampler_args(
self,
process_ind: int,
total_processes: int,
devices: Optional[List[int]] = None,
seeds: Optional[List[int]] = None,
deterministic_cudnn: bool = False,
) -> Dict[str, Any]:
config = self.TRAIN_CONFIGS[process_ind]
return {
"env_config":
config,
"max_steps":
self.MAX_STEPS,
"sensors":
self.SENSORS,
"action_space":
gym.spaces.Discrete(len(PointNavTask.class_action_names())),
"distance_to_goal":
self.DISTANCE_TO_GOAL, # type:ignore
}
def valid_task_sampler_args(
self,
process_ind: int,
total_processes: int,
devices: Optional[List[int]] = None,
seeds: Optional[List[int]] = None,
deterministic_cudnn: bool = False,
) -> Dict[str, Any]:
config = self.CONFIG.clone()
config.defrost()
config.DATASET.DATA_PATH = self.VALID_SCENES
config.MODE = "validate"
config.freeze()
return {
"env_config":
config,
"max_steps":
self.MAX_STEPS,
"sensors":
self.SENSORS,
"action_space":
gym.spaces.Discrete(len(PointNavTask.class_action_names())),
"distance_to_goal":
self.DISTANCE_TO_GOAL, # type:ignore
}
def test_task_sampler_args(
self,
process_ind: int,
total_processes: int,
devices: Optional[List[int]] = None,
seeds: Optional[List[int]] = None,
deterministic_cudnn: bool = False,
) -> Dict[str, Any]:
raise NotImplementedError("Testing not implemented for this tutorial.")