def test_dataset_splitting(split):
dataset_config = get_config(CFG_TEST).DATASET
dataset_config.defrost()
dataset_config.SPLIT = split
if not ObjectNavDatasetV1.check_config_paths_exist(dataset_config):
pytest.skip("Test skipped as dataset files are missing.")
scenes = ObjectNavDatasetV1.get_scenes_to_load(config=dataset_config)
assert (len(scenes) >
0), "Expected dataset contains separate episode file per scene."
dataset_config.CONTENT_SCENES = scenes[:PARTIAL_LOAD_SCENES]
full_dataset = make_dataset(id_dataset=dataset_config.TYPE,
config=dataset_config)
full_episodes = {(ep.scene_id, ep.episode_id)
for ep in full_dataset.episodes}
dataset_config.CONTENT_SCENES = scenes[:PARTIAL_LOAD_SCENES // 2]
split1_dataset = make_dataset(id_dataset=dataset_config.TYPE,
config=dataset_config)
split1_episodes = {(ep.scene_id, ep.episode_id)
for ep in split1_dataset.episodes}
dataset_config.CONTENT_SCENES = scenes[PARTIAL_LOAD_SCENES //
2:PARTIAL_LOAD_SCENES]
split2_dataset = make_dataset(id_dataset=dataset_config.TYPE,
config=dataset_config)
split2_episodes = {(ep.scene_id, ep.episode_id)
for ep in split2_dataset.episodes}
assert full_episodes == split1_episodes.union(
split2_episodes), "Split dataset is not equal to full dataset"
assert (len(split1_episodes.intersection(split2_episodes)) == 0
), "Intersection of split datasets is not the empty set"
def __init__(self, par, seq_len, config_file, action_list):
self.config_file = '{}/{}'.format(par.habitat_root, config_file)
self.seq_len = seq_len
self.actions = action_list
self.dets_nClasses = par.dets_nClasses
config = get_config(self.config_file)
config.defrost()
config.SIMULATOR.DEPTH_SENSOR.NORMALIZE_DEPTH = False
config.TASK.MEASUREMENTS.append('COLLISIONS')
config.freeze()
self.hfov = float(config.SIMULATOR.DEPTH_SENSOR.HFOV) * np.pi / 180.
dataset = make_dataset(id_dataset=config.DATASET.TYPE,
config=config.DATASET)
self.env = habitat.Env(config=config, dataset=dataset)
self.cropSize = par.crop_size
self.cropSizeObsv = par.crop_size_obsv
self.orig_res = (config.SIMULATOR.DEPTH_SENSOR.HEIGHT,
config.SIMULATOR.DEPTH_SENSOR.WIDTH)
self.normalize = True
self.pixFormat = 'NCHW'
self.dg = myDistanceToGoal(self.env.sim, config)
def test_mp3d_eqa_sim():
eqa_config = get_config(CFG_TEST)
if not mp3d_dataset.Matterport3dDatasetV1.check_config_paths_exist(
eqa_config.DATASET
):
pytest.skip("Please download Matterport3D EQA dataset to data folder.")
dataset = make_dataset(
id_dataset=eqa_config.DATASET.TYPE, config=eqa_config.DATASET
)
with habitat.Env(config=eqa_config, dataset=dataset) as env:
env.episodes = dataset.episodes[:EPISODES_LIMIT]
env.reset()
while not env.episode_over:
obs = env.step(env.task.action_space.sample())
if not env.episode_over:
assert "rgb" in obs, "RGB image is missing in observation."
assert obs["rgb"].shape[:2] == (
eqa_config.SIMULATOR.RGB_SENSOR.HEIGHT,
eqa_config.SIMULATOR.RGB_SENSOR.WIDTH,
), (
"Observation resolution {} doesn't correspond to config "
"({}, {}).".format(
obs["rgb"].shape[:2],
eqa_config.SIMULATOR.RGB_SENSOR.HEIGHT,
eqa_config.SIMULATOR.RGB_SENSOR.WIDTH,
)
)
def test_multiple_files_scene_path():
dataset_config = get_config(CFG_MULTI_TEST).DATASET
if not PointNavDatasetV1.check_config_paths_exist(dataset_config):
pytest.skip("Test skipped as dataset files are missing.")
scenes = PointNavDatasetV1.get_scenes_to_load(config=dataset_config)
assert (
len(scenes) > 0
), "Expected dataset contains separate episode file per scene."
dataset_config.defrost()
dataset_config.CONTENT_SCENES = scenes[:PARTIAL_LOAD_SCENES]
dataset_config.SCENES_DIR = os.path.join(
os.getcwd(), DEFAULT_SCENE_PATH_PREFIX
)
dataset_config.freeze()
partial_dataset = make_dataset(
id_dataset=dataset_config.TYPE, config=dataset_config
)
assert (
len(partial_dataset.scene_ids) == PARTIAL_LOAD_SCENES
), "Number of loaded scenes doesn't correspond."
print(partial_dataset.episodes[0].scene_id)
assert os.path.exists(
partial_dataset.episodes[0].scene_id
), "Scene file {} doesn't exist using absolute path".format(
partial_dataset.episodes[0].scene_id
)
def test_object_nav_task():
config = get_config(CFG_TEST)
if not ObjectNavDatasetV1.check_config_paths_exist(config.DATASET):
pytest.skip(
"Please download Matterport3D scene and ObjectNav Datasets to data folder."
)
dataset = make_dataset(id_dataset=config.DATASET.TYPE,
config=config.DATASET)
with habitat.Env(config=config, dataset=dataset) as env:
for i in range(10):
env.reset()
while not env.episode_over:
action = env.action_space.sample()
habitat.logger.info(f"Action : "
f"{action['action']}, "
f"args: {action['action_args']}.")
env.step(action)
metrics = env.get_metrics()
logger.info(metrics)
with pytest.raises(AssertionError):
env.step({"action": MoveForwardAction.name})
def make_env_fn(config: Config, env_class: Type[Union[Env, RLEnv]],
rank: int) -> Union[Env, RLEnv]:
r"""Creates an env of type env_class with specified config and rank.
This is to be passed in as an argument when creating VectorEnv.
Args:
config: root exp config that has core env config node as well as
env-specific config node.
env_class: class type of the env to be created.
rank: rank of env to be created (for seeding).
Returns:
env object created according to specification.
"""
if not config.USE_SYNC_VECENV:
level = logging.DEBUG if config.DEBUG else logging.INFO
logging.basicConfig(level=level,
format='%(asctime)s, %(levelname)s: %(message)s',
datefmt="%Y-%m-%d %H:%M:%S")
random.seed(rank)
np.random.seed(rank)
torch.manual_seed(rank)
dataset = make_dataset(config.TASK_CONFIG.DATASET.TYPE,
config=config.TASK_CONFIG.DATASET)
env = env_class(config=config, dataset=dataset)
env.seed(rank)
return env
def __init__(self, par, seq_len, config_file):
self.config_file = '{}/{}'.format(par.habitat_root, config_file)
self.seq_len = seq_len
self.dets_nClasses = par.dets_nClasses
config = get_config(self.config_file)
config.defrost()
config.SIMULATOR.DEPTH_SENSOR.NORMALIZE_DEPTH = False
config.freeze()
self.hfov = float(config.SIMULATOR.DEPTH_SENSOR.HFOV) * np.pi / 180.
#self.intr = np.zeros((4), dtype=np.float32)
#self.intr[0] = 1./np.tan(self.hfov/2.)*(config.SIMULATOR.DEPTH_SENSOR.WIDTH/2.) # fx
#self.intr[1] = 1./np.tan(self.hfov/2.)*(config.SIMULATOR.DEPTH_SENSOR.HEIGHT/2.) # fy
#self.intr[2] = config.SIMULATOR.DEPTH_SENSOR.WIDTH/2. # cx
#self.intr[3] = config.SIMULATOR.DEPTH_SENSOR.HEIGHT/2. # cy
dataset = make_dataset(id_dataset=config.DATASET.TYPE,
config=config.DATASET)
self.env = habitat.Env(config=config, dataset=dataset)
self.orig_res = (config.SIMULATOR.DEPTH_SENSOR.HEIGHT,
config.SIMULATOR.DEPTH_SENSOR.WIDTH)
self.cropSize = par.crop_size
self.normalize = True
self.pixFormat = 'NCHW'
def __init__(self,
config: Config,
dataset: Optional[Dataset] = None) -> None:
assert config.is_frozen(), ("Freeze the config before creating the "
"environment, use config.freeze()")
self._config = config
self._dataset = dataset
self._current_episode_index = None
if self._dataset is None and config.DATASET.TYPE:
self._dataset = make_dataset(id_dataset=config.DATASET.TYPE,
config=config.DATASET)
self._episodes = self._dataset.episodes if self._dataset else []
self._sim = make_sim(id_sim=self._config.SIMULATOR.TYPE,
config=self._config.SIMULATOR)
self._task = make_task(
self._config.TASK.TYPE,
task_config=self._config.TASK,
sim=self._sim,
dataset=dataset,
)
self.observation_space = SpaceDict({
**self._sim.sensor_suite.observation_spaces.spaces,
**self._task.sensor_suite.observation_spaces.spaces,
})
self.action_space = self._sim.action_space
self._max_episode_seconds = (
self._config.ENVIRONMENT.MAX_EPISODE_SECONDS)
self._max_episode_steps = self._config.ENVIRONMENT.MAX_EPISODE_STEPS
self._elapsed_steps = 0
self._episode_start_time: Optional[float] = None
self._episode_over = False
def test_dataset_splitting(split):
dataset_config = get_config(CFG_TEST).DATASET
dataset_config.defrost()
dataset_config.SPLIT = split
if not r2r_vln_dataset.VLNDatasetV1.check_config_paths_exist(
dataset_config
):
pytest.skip("Please download Matterport3D R2R dataset to data folder.")
scenes = r2r_vln_dataset.VLNDatasetV1.get_scenes_to_load(
config=dataset_config
)
assert (
len(scenes) > 0
), "Expected dataset contains separate episode file per scene."
dataset_config.CONTENT_SCENES = scenes
full_dataset = make_dataset(
id_dataset=dataset_config.TYPE, config=dataset_config
)
full_episodes = {
(ep.scene_id, ep.episode_id) for ep in full_dataset.episodes
}
dataset_config.CONTENT_SCENES = scenes[0 : len(scenes) // 2]
split1_dataset = make_dataset(
id_dataset=dataset_config.TYPE, config=dataset_config
)
split1_episodes = {
(ep.scene_id, ep.episode_id) for ep in split1_dataset.episodes
}
dataset_config.CONTENT_SCENES = scenes[len(scenes) // 2 :]
split2_dataset = make_dataset(
id_dataset=dataset_config.TYPE, config=dataset_config
)
split2_episodes = {
(ep.scene_id, ep.episode_id) for ep in split2_dataset.episodes
}
assert full_episodes == split1_episodes.union(
split2_episodes
), "Split dataset is not equal to full dataset"
assert (
len(split1_episodes.intersection(split2_episodes)) == 0
), "Intersection of split datasets is not the empty set"
def __init__(self,
config: Config,
dataset: Optional[Dataset] = None) -> None:
"""Constructor
:param config: config for the environment. Should contain id for
simulator and ``task_name`` which are passed into ``make_sim`` and
``make_task``.
:param dataset: reference to dataset for task instance level
information. Can be defined as :py:`None` in which case
``_episodes`` should be populated from outside.
"""
assert config.is_frozen(), ("Freeze the config before creating the "
"environment, use config.freeze().")
self._config = config
self._dataset = dataset
self._current_episode_index = None
if self._dataset is None and config.DATASET.TYPE:
self._dataset = make_dataset(id_dataset=config.DATASET.TYPE,
config=config.DATASET)
self._episodes = self._dataset.episodes if self._dataset else []
self._current_episode = None
iter_option_dict = {
k.lower(): v
for k, v in config.ENVIRONMENT.ITERATOR_OPTIONS.items()
}
self._episode_iterator = self._dataset.get_episode_iterator(
**iter_option_dict)
# load the first scene if dataset is present
if self._dataset:
assert (len(self._dataset.episodes) >
0), "dataset should have non-empty episodes list"
self._config.defrost()
self._config.SIMULATOR.SCENE = self._dataset.episodes[0].scene_id
self._config.freeze()
self._sim = make_sim(id_sim=self._config.SIMULATOR.TYPE,
config=self._config.SIMULATOR)
self._task = make_task(
self._config.TASK.TYPE,
config=self._config.TASK,
sim=self._sim,
dataset=self._dataset,
)
self.observation_space = SpaceDict({
**self._sim.sensor_suite.observation_spaces.spaces,
**self._task.sensor_suite.observation_spaces.spaces,
})
self.action_space = self._task.action_space
self._max_episode_seconds = (
self._config.ENVIRONMENT.MAX_EPISODE_SECONDS)
self._max_episode_steps = self._config.ENVIRONMENT.MAX_EPISODE_STEPS
self._elapsed_steps = 0
self._episode_start_time: Optional[float] = None
self._episode_over = False
def test_r2r_vln_sim():
vln_config = get_config(CFG_TEST)
if not r2r_vln_dataset.VLNDatasetV1.check_config_paths_exist(
vln_config.DATASET
):
pytest.skip(
"Please download Matterport3D R2R VLN dataset to data folder."
)
dataset = make_dataset(
id_dataset=vln_config.DATASET.TYPE, config=vln_config.DATASET
)
env = habitat.Env(config=vln_config, dataset=dataset)
env.episodes = dataset.episodes[:EPISODES_LIMIT]
follower = ShortestPathFollower(
env.sim, goal_radius=0.5, return_one_hot=False
)
assert env
for i in range(len(env.episodes)):
env.reset()
path = env.current_episode.reference_path + [
env.current_episode.goals[0].position
]
for point in path:
done = False
while not done:
best_action = follower.get_next_action(point)
if best_action == None:
break
obs = env.step(best_action)
assert "rgb" in obs, "RGB image is missing in observation."
assert (
"instruction" in obs
), "Instruction is missing in observation."
assert (
obs["instruction"]["text"]
== env.current_episode.instruction.instruction_text
), "Instruction from sensor does not match the intruction from the episode"
assert obs["rgb"].shape[:2] == (
vln_config.SIMULATOR.RGB_SENSOR.HEIGHT,
vln_config.SIMULATOR.RGB_SENSOR.WIDTH,
), (
"Observation resolution {} doesn't correspond to config "
"({}, {}).".format(
obs["rgb"].shape[:2],
vln_config.SIMULATOR.RGB_SENSOR.HEIGHT,
vln_config.SIMULATOR.RGB_SENSOR.WIDTH,
)
)
env.close()
def test_r2r_vln_dataset():
vln_config = get_config(CFG_TEST)
if not r2r_vln_dataset.VLNDatasetV1.check_config_paths_exist(
vln_config.DATASET):
pytest.skip("Please download Matterport3D R2R dataset to data folder.")
dataset = make_dataset(id_dataset=vln_config.DATASET.TYPE,
config=vln_config.DATASET)
assert dataset
assert (len(dataset.episodes) == R2R_VAL_SEEN_EPISODES
), "Val Seen split episode number mismatch"
check_json_serializaiton(dataset)
def test_multiple_files_pointnav_dataset(cfg_path: str):
dataset_config = get_config(cfg_path).DATASET
if not PointNavDatasetV1.check_config_paths_exist(dataset_config):
pytest.skip("Test skipped as dataset files are missing.")
scenes = PointNavDatasetV1.get_scenes_to_load(config=dataset_config)
assert (len(scenes) >
0), "Expected dataset contains separate episode file per scene."
dataset_config.defrost()
dataset_config.CONTENT_SCENES = scenes
dataset_config.freeze()
partial_dataset = make_dataset(id_dataset=dataset_config.TYPE,
config=dataset_config)
check_json_serializaiton(partial_dataset)
def get_eval_dataset(shell_args, data_subset="val"):
if shell_args.dataset == "mp3d":
data_path = "data/datasets/pointnav/mp3d/v1/{split}/{split}.json.gz"
elif shell_args.dataset == "gibson":
data_path = "data/datasets/pointnav/gibson/v1/{split}/{split}.json.gz"
else:
raise NotImplementedError("No rule for this dataset.")
config = get_dataset_config(data_path, data_subset,
shell_args.max_episode_length, 0, [], [])
dataset = make_dataset(config.DATASET.TYPE, config=config.DATASET)
assert len(dataset.episodes) > 0, "empty datasets"
return dataset
def __init__(
self,
gpu_id: int,
unique_dataset_name: str,
split: str,
data_path: str,
images_before_reset: int,
sensors: Optional[List[str]] = None,
resolution: Optional[int] = 256,
) -> None:
if sensors is None:
sensors = ["RGB_SENSOR", "DEPTH_SENSOR"]
self.images_before_reset = images_before_reset
config = make_config(gpu_id, split, data_path, sensors, resolution)
data_dir = os.path.join("data/scene_episodes/",
unique_dataset_name + "_" + split)
self.dataset_name = config.DATASET.TYPE
if not os.path.exists(data_dir):
os.makedirs(data_dir)
data_path = os.path.join(data_dir, "dataset_one_ep_per_scene.json.gz")
# Creates a dataset where each episode is a random spawn point in each scene.
if not os.path.exists(data_path):
dataset = make_dataset(config.DATASET.TYPE, config=config.DATASET)
# Get one episode per scene in dataset
scene_episodes = {}
for episode in dataset.episodes:
if episode.scene_id not in scene_episodes:
scene_episodes[episode.scene_id] = episode
scene_episodes = list(scene_episodes.values())
print("scene episodes", len(scene_episodes))
dataset.episodes = scene_episodes
if not os.path.exists(data_path):
# Multiproc do check again before write.
json = dataset.to_json().encode("utf-8")
with gzip.GzipFile(data_path, "w") as fout:
fout.write(json)
dataset = mp3d_dataset.PointNavDatasetV1()
with gzip.open(data_path, "rt") as f:
dataset.from_json(f.read())
config.TASK.SENSORS = ["POINTGOAL_SENSOR"]
if "SEMANTIC_SESNOR" in sensors:
config.TASK.SENSOR.append("CLASS_SEGMENTATION_SENSOR")
config.TASK.CLASS_SEGMENTATION_SENSOR.HEIGHT = config.TASK.HEIGHT
config.TASK.CLASS_SEGMENTATION_SENSOR.WIDTH = config.TASK.WIDTH
config.freeze()
self.env = habitat.Env(config=config, dataset=dataset)
random.shuffle(self.env.episodes)
self.num_samples = 0
def test_multiple_files_pointnav_dataset():
dataset_config = get_config(CFG_MULTI_TEST).DATASET
if not PointNavDatasetV1.check_config_paths_exist(dataset_config):
pytest.skip("Test skipped as dataset files are missing.")
scenes = PointNavDatasetV1.get_scenes_to_load(config=dataset_config)
assert (len(scenes) >
0), "Expected dataset contains separate episode file per scene."
dataset_config.defrost()
dataset_config.CONTENT_SCENES = scenes[:PARTIAL_LOAD_SCENES]
dataset_config.freeze()
partial_dataset = make_dataset(id_dataset=dataset_config.TYPE,
config=dataset_config)
assert (len(partial_dataset.scene_ids) == PARTIAL_LOAD_SCENES
), "Number of loaded scenes doesn't correspond."
check_json_serializaiton(partial_dataset)
def __init__(self,
config: Config,
dataset: Optional[Dataset] = None) -> None:
assert config.is_frozen(), ("Freeze the config before creating the "
"environment, use config.freeze().")
self._config = config
self._dataset = dataset
self._current_episode_index = None
if self._dataset is None and config.DATASET.TYPE:
self._dataset = make_dataset(id_dataset=config.DATASET.TYPE,
config=config.DATASET)
self._episodes = self._dataset.episodes if self._dataset else []
self._current_episode = None
iter_option_dict = {
k.lower(): v
for k, v in config.ENVIRONMENT.ITERATOR_OPTIONS.items()
}
self._episode_iterator = self._dataset.get_episode_iterator(
**iter_option_dict)
# load the first scene if dataset is present
if self._dataset:
assert (len(self._dataset.episodes) >
0), "dataset should have non-empty episodes list"
self._config.defrost()
self._config.SIMULATOR.SCENE = self._dataset.episodes[0].scene_id
self._config.freeze()
self._sim = make_sim(id_sim=self._config.SIMULATOR.TYPE,
config=self._config.SIMULATOR)
self._task = make_task(
self._config.TASK.TYPE,
task_config=self._config.TASK,
sim=self._sim,
dataset=self._dataset,
)
self.observation_space = SpaceDict({
**self._sim.sensor_suite.observation_spaces.spaces,
**self._task.sensor_suite.observation_spaces.spaces,
})
self.action_space = self._sim.action_space
self._max_episode_seconds = (
self._config.ENVIRONMENT.MAX_EPISODE_SECONDS)
self._max_episode_steps = self._config.ENVIRONMENT.MAX_EPISODE_STEPS
self._elapsed_steps = 0
self._episode_start_time: Optional[float] = None
self._episode_over = False
def test_eqa_task():
eqa_config = get_config(CFG_TEST)
if not mp3d_dataset.Matterport3dDatasetV1.check_config_paths_exist(
eqa_config.DATASET
):
pytest.skip("Please download Matterport3D EQA dataset to data folder.")
dataset = make_dataset(
id_dataset=eqa_config.DATASET.TYPE, config=eqa_config.DATASET
)
with habitat.Env(config=eqa_config, dataset=dataset) as env:
env.episodes = list(
filter(
lambda e: int(e.episode_id)
in TEST_EPISODE_SET[:EPISODES_LIMIT],
dataset.episodes,
)
)
env.reset()
for i in range(10):
action = sample_non_stop_action(env.action_space)
if action["action"] != AnswerAction.name:
env.step(action)
metrics = env.get_metrics()
del metrics["episode_info"]
logger.info(metrics)
correct_answer_id = env.current_episode.question.answer_token
env.step(
{
"action": AnswerAction.name,
"action_args": {"answer_id": correct_answer_id},
}
)
metrics = env.get_metrics()
del metrics["episode_info"]
logger.info(metrics)
assert metrics["answer_accuracy"] == 1
with pytest.raises(AssertionError):
env.step({"action": MoveForwardAction.name})
def __init__(self, task_config: Optional[Config] = None) -> None:
r"""..
:param task_config: config to be used for creating the environment
"""
dummy_config = Config()
dummy_config.RL = Config()
dummy_config.RL.SLACK_REWARD = -0.01
dummy_config.RL.SUCCESS_REWARD = 10
dummy_config.RL.WITH_TIME_PENALTY = True
dummy_config.RL.DISTANCE_REWARD_SCALE = 1
dummy_config.RL.WITH_DISTANCE_REWARD = True
dummy_config.RL.defrost()
dummy_config.TASK_CONFIG = task_config
dummy_config.freeze()
dataset = make_dataset(id_dataset=task_config.DATASET.TYPE,
config=task_config.DATASET)
self._env = NavRLEnv(config=dummy_config, dataset=dataset)
def setup(self, create_decoder=True):
assert self.shell_args.update_policy_decoder_features or self.shell_args.update_encoder_features
super(HabitatRLTrainAndEvalRunner, self).setup(create_decoder)
self.shell_args.cuda = not self.shell_args.no_cuda and torch.cuda.is_available(
)
print("Starting make dataset")
start_t = time.time()
config = self.configs[0]
dataset = make_dataset(config.DATASET.TYPE, config=config.DATASET)
observation_shape_chw = (3, config.SIMULATOR.RGB_SENSOR.HEIGHT,
config.SIMULATOR.RGB_SENSOR.WIDTH)
print("made dataset")
assert len(dataset.episodes) > 0, "empty datasets"
if self.shell_args.num_train_scenes > 0:
scene_ids = sorted(dataset.scene_ids)
random.seed(0)
random.shuffle(scene_ids)
used_scene_ids = set(scene_ids[:self.shell_args.num_train_scenes])
dataset.filter_episodes(lambda x: x.scene_id in used_scene_ids)
if self.shell_args.record_video:
random.shuffle(dataset.episodes)
datasets = dataset.get_splits(self.shell_args.num_processes,
remove_unused_episodes=True,
collate_scene_ids=True)
print("Dataset creation time %.3f" % (time.time() - start_t))
self.rollouts = RolloutStorageWithMultipleObservations(
self.shell_args.num_forward_rollout_steps,
self.shell_args.num_processes,
observation_shape_chw,
self.gym_action_space,
self.agent.recurrent_hidden_state_size,
self.observation_space,
"rgb",
)
self.rollouts.to(self.device)
print("Feeding dummy batch")
dummy_start = time.time()
self.optimizer.update(self.rollouts, self.shell_args)
print("Done feeding dummy batch %.3f" % (time.time() - dummy_start))
self.logger = None
if self.shell_args.tensorboard:
self.logger = tensorboard_logger.Logger(
os.path.join(self.shell_args.log_prefix,
self.shell_args.tensorboard_dirname,
self.time_str + "_train"))
self.datasets = {"train": datasets, "val": self.eval_datasets}
self.train_stats = dict(
num_episodes=np.zeros(self.shell_args.num_processes,
dtype=np.int32),
num_steps=np.zeros(self.shell_args.num_processes, dtype=np.int32),
reward=np.zeros(self.shell_args.num_processes, dtype=np.float32),
spl=np.zeros(self.shell_args.num_processes, dtype=np.float32),
visited_states=np.zeros(self.shell_args.num_processes,
dtype=np.int32),
success=np.zeros(self.shell_args.num_processes, dtype=np.int32),
end_geodesic_distance=np.zeros(self.shell_args.num_processes,
dtype=np.float32),
start_geodesic_distance=np.zeros(self.shell_args.num_processes,
dtype=np.float32),
delta_geodesic_distance=np.zeros(self.shell_args.num_processes,
dtype=np.float32),
distance_from_start=np.zeros(self.shell_args.num_processes,
dtype=np.float32),
)
def construct_envs(config: Config, env_class: Type[Union[Env,
RLEnv]]) -> VectorEnv:
r"""Create VectorEnv object with specified config and env class type.
To allow better performance, dataset are split into small ones for
each individual env, grouped by scenes.
Args:
config: configs that contain num_processes as well as information
necessary to create individual environments.
env_class: class type of the envs to be created
Returns:
VectorEnv object created according to specification.
"""
num_processes = config.NUM_PROCESSES
configs = []
env_classes = [env_class for _ in range(num_processes)]
dataset = make_dataset(config.TASK_CONFIG.DATASET.TYPE)
scenes = dataset.get_scenes_to_load(config.TASK_CONFIG.DATASET)
# rearrange scenes in the order of scene size since there is a severe imbalance of data size
if "replica" in config.TASK_CONFIG.DATASET.SCENES_DIR:
scenes_new = list()
for scene in SCENES:
if scene in scenes:
scenes_new.append(scene)
scenes = scenes_new
if len(scenes) > 0:
# random.shuffle(scenes)
assert len(scenes) >= num_processes, (
"reduce the number of processes as there "
"aren't enough number of scenes")
scene_splits = [[] for _ in range(num_processes)]
for idx, scene in enumerate(scenes):
scene_splits[idx % len(scene_splits)].append(scene)
assert sum(map(len, scene_splits)) == len(scenes)
for i in range(num_processes):
task_config = config.TASK_CONFIG.clone()
task_config.defrost()
if len(scenes) > 0:
task_config.DATASET.CONTENT_SCENES = scene_splits[i]
logging.debug('All scenes: {}'.format(','.join(scene_splits[i])))
# overwrite the task config with top-level config file
task_config.SIMULATOR.HABITAT_SIM_V0.GPU_DEVICE_ID = (
config.SIMULATOR_GPU_ID)
task_config.SIMULATOR.AGENT_0.SENSORS = config.SENSORS
task_config.freeze()
config.defrost()
config.TASK_CONFIG = task_config
config.freeze()
configs.append(config.clone())
# use VectorEnv for the best performance and ThreadedVectorEnv for debugging
if config.USE_SYNC_VECENV:
env_launcher = SyncVectorEnv
logging.info('Using SyncVectorEnv')
elif config.USE_VECENV:
env_launcher = habitat.VectorEnv
logging.info('Using VectorEnv')
else:
env_launcher = habitat.ThreadedVectorEnv
logging.info('Using ThreadedVectorEnv')
envs = env_launcher(
make_env_fn=make_env_fn,
env_fn_args=tuple(
tuple(zip(configs, env_classes, range(num_processes)))),
)
return envs
def __init__(self,
config: Config,
dataset: Optional[Dataset] = None) -> None:
"""Constructor. Mimics the :ref:`Env` constructor.
Args:
:param config: config for the environment. Should contain id for
simulator and ``task_name`` for each task, which are passed into ``make_sim`` and
``make_task``.
:param dataset: reference to dataset used for first task instance level
information. Can be defined as :py:`None` in which case
dataset will be built using ``make_dataset`` and ``config``.
"""
# let superclass instantiate current task by merging first task in TASKS to TASK
if len(config.MULTI_TASK.TASKS):
logger.info(
"Overwriting config.TASK ({}) with first entry in config.MULTI_TASK.TASKS ({})."
.format(config.TASK.TYPE, config.MULTI_TASK.TASKS[0].TYPE))
config.defrost()
config.TASK.merge_from_other_cfg(config.MULTI_TASK.TASKS[0])
config.freeze()
# TASKS[0] dataset has higher priority over default one (if specified), instatiate it before
if dataset is None and config.MULTI_TASK.TASKS[0].DATASET.TYPE:
dataset = make_dataset(
id_dataset=config.MULTI_TASK.TASKS[0].DATASET.TYPE,
config=config.MULTI_TASK.TASKS[0].DATASET,
)
# initialize first task leveraging Env
super().__init__(config, dataset=dataset)
# instatiate other tasks
self._tasks = [self._task]
self._curr_task_idx = 0
# keep each tasks episode iterator to avoid re-creation
self._episode_iterators = [self._episode_iterator]
for task in config.MULTI_TASK.TASKS[1:]:
self._tasks.append(
make_task(
task.TYPE,
config=task,
sim=self._sim,
# each task gets its dataset
dataset=make_dataset( # TODO: lazy make_dataset support
id_dataset=task.DATASET.TYPE,
config=task.DATASET),
))
# get task episode iterator
iter_option_dict = {
k.lower(): v
for k, v in task.EPISODE_ITERATOR_OPTIONS.items()
}
iter_option_dict["seed"] = self._config.SEED
task_ep_iterator: EpisodeIterator = self._tasks[
-1]._dataset.get_episode_iterator(**iter_option_dict)
self._episode_iterators.append(task_ep_iterator)
# episode counter
self._eps_counter = -1
self._cumulative_steps_counter = 0
# when and how to change task is defined here
self._task_sampling_behavior = (
config.MULTI_TASK.TASK_ITERATOR.TASK_SAMPLING)
self._change_task_behavior = (
config.MULTI_TASK.TASK_ITERATOR.TASK_CHANGE_TIMESTEP)
# custom task label can be specified
self._curr_task_label = self._task._config.get("TASK_LABEL",
self._curr_task_idx)
# add task_idx to observation space
self.observation_space = spaces.Dict({
**self._sim.sensor_suite.observation_spaces.spaces,
**self._task.sensor_suite.observation_spaces.spaces,
"task_idx":
spaces.Discrete(len(self._tasks)),
})
[])
config.defrost()
for sensor in config.SIMULATOR.AGENT_0.SENSORS:
config.SIMULATOR[sensor].HEIGHT = RESOLUTION
config.SIMULATOR[sensor].WIDTH = RESOLUTION
config.TASK.COLLISION_REWARD = 0 # -0.1
config.ENVIRONMENT.MAX_EPISODE_STEPS = 250
config.TASK.TOP_DOWN_MAP.DRAW_SOURCE_AND_TARGET = False
config.TASK.NUM_EPISODES_BEFORE_JUMP = -1
config.TASK.GRID_SIZE = 1
config.TASK.NEW_GRID_CELL_REWARD = 1
config.TASK.RETURN_VISITED_GRID = False
config.freeze()
dataset = make_dataset(config.DATASET.TYPE, config=config.DATASET)
datasets = {"val": dataset}
env = ExplorationRLEnv(config=config, datasets=datasets)
goal_radius = env.episodes[0].goals[0].radius
if goal_radius is None:
goal_radius = config.SIMULATOR.FORWARD_STEP_SIZE
sim = env.habitat_env.sim
follower = ShortestPathFollower(sim, goal_radius, False)
follower.mode = "geodesic_path"
max_dists = []
for _ in tqdm.tqdm(range(len(env.episodes))):
env.reset()
points = []
dists = []
def test_mp3d_eqa_sim_correspondence():
eqa_config = get_config(CFG_TEST)
if not mp3d_dataset.Matterport3dDatasetV1.check_config_paths_exist(
eqa_config.DATASET
):
pytest.skip(
"Please download Matterport3D EQA dataset to " "data folder."
)
dataset = make_dataset(
id_dataset=eqa_config.DATASET.TYPE, config=eqa_config.DATASET
)
env = habitat.Env(config=eqa_config, dataset=dataset)
env.episodes = [
episode
for episode in dataset.episodes
if int(episode.episode_id) in TEST_EPISODE_SET[:EPISODES_LIMIT]
]
ep_i = 0
cycles_n = 2
while cycles_n > 0:
env.reset()
episode = env.current_episode
assert (
len(episode.goals) == 1
), "Episode has no goals or more than one."
assert (
len(episode.shortest_paths) == 1
), "Episode has no shortest paths or more than one."
start_state = env.sim.get_agent_state()
assert np.allclose(
start_state.position, episode.start_position
), "Agent's start position diverges from the shortest path's one."
rgb_mean = 0
logger.info(
"{id} {question}\n{answer}".format(
id=episode.episode_id,
question=episode.question.question_text,
answer=episode.question.answer_text,
)
)
for step_id, point in enumerate(episode.shortest_paths[0]):
cur_state = env.sim.get_agent_state()
logger.info(
"diff position: {} diff rotation: {} "
"cur_state.position: {} shortest_path.position: {} "
"cur_state.rotation: {} shortest_path.rotation: {} action: {}"
"".format(
cur_state.position - point.position,
cur_state.rotation - point.rotation,
cur_state.position,
point.position,
cur_state.rotation,
point.rotation,
point.action,
)
)
assert np.allclose(
[cur_state.position[0], cur_state.position[2]],
[point.position[0], point.position[2]],
atol=CLOSE_STEP_THRESHOLD * (step_id + 1),
), "Agent's path diverges from the shortest path."
obs = env.step(point.action)
if not env.episode_over:
rgb_mean += obs["rgb"][:, :, :3].mean()
if ep_i < len(RGB_EPISODE_MEANS):
rgb_mean = rgb_mean / len(episode.shortest_paths[0])
assert np.isclose(
RGB_EPISODE_MEANS[int(episode.episode_id)], rgb_mean
), "RGB output doesn't match the ground truth."
ep_i = (ep_i + 1) % EPISODES_LIMIT
if ep_i == 0:
cycles_n -= 1
env.close()
def main():
parser = argparse.ArgumentParser()
# parser.add_argument('--sound', default=False, action='store_true')
parser.add_argument(
"--run-type",
choices=["train", "eval"],
default='eval',
help="run type of the experiment (train or eval)",
)
parser.add_argument(
"--exp-config",
type=str,
# required=True,
default='baselines/config/audiogoal_rgb_demo.yaml',
help="path to config yaml containing info about experiment",
)
parser.add_argument(
"opts",
default=None,
nargs=argparse.REMAINDER,
help="Modify config options from command line",
)
parser.add_argument(
"--debug",
default=False,
action='store_true',
help="Modify config options from command line",
)
parser.add_argument(
"--keys",
default='',
type=str,
help="Modify config options from command line",
)
args = parser.parse_args()
# file_handler = logging.FileHandler(log_file, mode=mode)
stdout_handler = logging.StreamHandler(sys.stdout)
level = logging.INFO if not args.debug else logging.DEBUG
logging.basicConfig(level=level,
handlers=[stdout_handler],
format='%(asctime)s, %(levelname)s: %(message)s',
datefmt="%Y-%m-%d %H:%M:%S")
config = get_config(config_paths=args.exp_config,
opts=args.opts,
run_type=args.run_type)
config.defrost()
config.TASK_CONFIG.TASK.MEASUREMENTS.append("TOP_DOWN_MAP")
config.TASK_CONFIG.DATASET.SPLIT = config.EVAL.SPLIT
if args.keys == '':
config.TASK_CONFIG.SIMULATOR.RGB_SENSOR.WIDTH = config.TASK_CONFIG.SIMULATOR.RGB_SENSOR.HEIGHT = \
config.TASK_CONFIG.SIMULATOR.DEPTH_SENSOR.WIDTH = config.TASK_CONFIG.SIMULATOR.DEPTH_SENSOR.HEIGHT = 256
config.TASK_CONFIG.SIMULATOR.CONTINUOUS_VIEW_CHANGE = False
else:
config.TASK_CONFIG.TASK.TOP_DOWN_MAP.DRAW_GOAL_POSITIONS = False
config.freeze()
print(config)
dataset = make_dataset(id_dataset=config.TASK_CONFIG.DATASET.TYPE,
config=config.TASK_CONFIG.DATASET)
env = NavRLEnv(config=config, dataset=dataset)
if args.keys == '':
interactive_demo(config, env)
else:
keys = args.keys.split(',')
following(config, env, keys)
import cv2
from env_utils.vistarget_nav_task import CustomVisTargetSensor
for split in splits:
config = get_config('IL_configs/base.yaml')
configs = []
habitat_api_path = os.path.join(os.path.dirname(habitat.__file__), '../')
config.defrost()
config.TASK_CONFIG.DATASET.SCENES_DIR = os.path.join(
habitat_api_path, config.TASK_CONFIG.DATASET.SCENES_DIR)
config.TASK_CONFIG.DATASET.DATA_PATH = os.path.join(
habitat_api_path, config.TASK_CONFIG.DATASET.DATA_PATH)
config.TASK_CONFIG.DATASET.SPLIT = split
config.freeze()
dataset = make_dataset('PointNav-v1')
scenes = config.TASK_CONFIG.DATASET.CONTENT_SCENES
if "*" in config.TASK_CONFIG.DATASET.CONTENT_SCENES:
scenes = dataset.get_scenes_to_load(config.TASK_CONFIG.DATASET)
# 17DRP5sb8fy 1LXtFkjw3qL 1pXnuDYAj8r 29hnd4uzFmX 2n8kARJN3HM 5LpN3gDmAk7 5q7pvUzZiYa 759xd9YjKW5 7y3sRwLe3Va 82sE5b5pLXE
# 8WUmhLawc2A B6ByNegPMKs D7G3Y4RVNrH D7N2EKCX4Sj E9uDoFAP3SH EDJbREhghzL GdvgFV5R1Z5 HxpKQynjfin JF19kD82Mey JeFG25nYj2p
# JmbYfDe2QKZ
valid_scenes = []
for scene_name in scenes:
if split == 'train':
scene_data_list = [x for x in train_data_list if scene_name in x]
elif split == 'val':
scene_data_list = [x for x in val_data_list if scene_name in x]
changed = False
for each_data in scene_data_list:
valid = ((time.time() - os.stat(each_data).st_mtime) / 3600) > 24
def __init__(self, split, gpu_id, envs_processed, envs_to_process, opts, vectorize=False, seed=0) -> None:
# self.vectorize = vectorize
self.vectorize = False
# print("gpu_id", gpu_id)
resolution = opts.W
if opts.no_sem_images and opts.no_txt_semantic and opts.no_binary_semantic:
sensors = ["RGB_SENSOR", "DEPTH_SENSOR"]
else:
sensors = ["RGB_SENSOR", "DEPTH_SENSOR", "SEMANTIC_SENSOR"]
if split == "train":
data_path = opts.train_data_path
elif split == "val":
data_path = opts.val_data_path
elif split == "test":
data_path = opts.test_data_path
else:
raise Exception("Invalid split")
unique_dataset_name = opts.dataset
self.num_parallel_envs = 1
self.images_before_reset = opts.images_before_reset
config = make_config(
opts.config,
gpu_id,
split,
data_path,
sensors,
resolution,
opts.scenes_dir,
)
data_dir = os.path.join(
"./util/scripts/mp3d_data_gen_deps/scene_episodes", unique_dataset_name + "_" + split
)
self.dataset_name = config.DATASET.TYPE
# print(data_dir)
if not os.path.exists(data_dir):
os.makedirs(data_dir)
data_path = os.path.join(data_dir, "dataset_one_ep_per_scene.json.gz")
# Creates a dataset where each episode is a random spawn point in each scene.
if not (os.path.exists(data_path)):
print("Creating dataset...", flush=True)
dataset = make_dataset(config.DATASET.TYPE, config=config.DATASET)
# Get one episode per scene in dataset
scene_episodes = {}
for episode in tqdm.tqdm(dataset.episodes):
if episode.scene_id not in scene_episodes:
scene_episodes[episode.scene_id] = episode
scene_episodes = list(scene_episodes.values())
dataset.episodes = scene_episodes
if not os.path.exists(data_path):
# Multiproc do check again before write.
json = dataset.to_json().encode("utf-8")
with gzip.GzipFile(data_path, "w") as fout:
fout.write(json)
print("Finished dataset...", flush=True)
# Load in data and update the location to the proper location (else
# get a weird, uninformative, error -- Affine2Dtransform())
dataset = mp3d_dataset.PointNavDatasetV1()
with gzip.open(data_path, "rt") as f:
dataset.from_json(f.read())
envs = []
for i in range(0, len(dataset.episodes)):
scene_id = dataset.episodes[i].scene_id.split("/")[-2]
# Check if scene already processed
if scene_id not in envs_processed:
# Check if user wants to process this scene (if no scene is specified then ignore this condition)
if len(envs_to_process) == 0 or scene_id in envs_to_process:
dataset.episodes[i].scene_id = dataset.episodes[i].scene_id.replace(
'/checkpoint/erikwijmans/data/mp3d/',
opts.scenes_dir + '/mp3d/')
envs.append(dataset.episodes[i])
dataset.episodes = envs
config.TASK.SENSORS = ["POINTGOAL_SENSOR"]
config.freeze()
self.rng = np.random.RandomState(seed)
# Now look at vector environments
if self.vectorize:
configs, datasets = _load_datasets(
(
opts.config,
gpu_id,
split,
data_path,
sensors,
resolution,
opts.scenes_dir,
),
dataset,
data_path,
opts.scenes_dir + '/mp3d/',
num_workers=self.num_parallel_envs,
)
num_envs = len(configs)
env_fn_args = tuple(zip(configs, datasets, range(num_envs)))
envs = habitat.VectorEnv(
env_fn_args=env_fn_args,
multiprocessing_start_method="forkserver",
)
self.env = envs
self.num_train_envs = int(0.9 * (self.num_parallel_envs))
self.num_val_envs = self.num_parallel_envs - self.num_train_envs
else:
self.env = habitat.Env(config=config, dataset=dataset) # TODO: End randomization here
self.env_sim = self.env.sim
self.rng.shuffle(self.env.episodes)
self.env_sim = self.env.sim
self.num_samples = 0
# Set up intrinsic parameters
self.hfov = config.SIMULATOR.DEPTH_SENSOR.HFOV * np.pi / 180.0
self.W = resolution
self.K = np.array(
[
[1.0 / np.tan(self.hfov / 2.0), 0.0, 0.0, 0.0],
[0, 1.0 / np.tan(self.hfov / 2.0), 0.0, 0.0],
[0.0, 0.0, 1.0, 0.0],
[0.0, 0.0, 0.0, 1.0],
],
dtype=np.float32,
)
self.invK = np.linalg.inv(self.K)
self.config = config
self.opts = opts
if self.opts.normalize_image:
self.transform = transforms.Compose(
[
transforms.ToTensor(),
transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]),
]
) # Using same normalization as BigGan
else:
self.transform = transforms.ToTensor()