def test_agent(agent):
r"""Runs experiment given mode and config
Args:
exp_config: path to config file.
run_type: "train" or "eval.
opts: list of strings of additional config options.
Returns:
None.
"""
config_file = r"/home/userone/workspace/bed1/ppo_custom/ppo_pointnav_example.yaml"
checkpoint_file = r"/home/userone/workspace/bed1/ppo_custom/tmp/ppo_transfer_rgb/new_checkpoints/ckpt.19.pth"
# config = get_config1(agent)
config = get_config(config_file, None)
config.defrost()
config.TASK_CONFIG.DATASET.DATA_PATH = config.DATA_PATH_SET
config.TASK_CONFIG.DATASET.SCENES_DIR = config.SCENE_DIR_SET
config.freeze()
random.seed(config.TASK_CONFIG.SEED)
np.random.seed(config.TASK_CONFIG.SEED)
if agent == "ppo_agent":
trainer = ppo_trainer.PPOTrainer(config)
elif agent == "ppo_rgb_transfer":
trainer = transfer_ppo.PPOTrainer(config)
trainer.eval(checkpoint_file)
def run_exp(exp_config: str, run_type: str, opts=None) -> None:
r"""Runs experiment given mode and config
Args:
exp_config: path to config file.
run_type: "train" or "eval.
opts: list of strings of additional config options.
Returns:
None.
"""
config = get_config(exp_config, opts)
random.seed(config.TASK_CONFIG.SEED)
np.random.seed(config.TASK_CONFIG.SEED)
torch.manual_seed(config.TASK_CONFIG.SEED)
trainer_init = baseline_registry.get_trainer(config.TRAINER_NAME)
assert trainer_init is not None, f"{config.TRAINER_NAME} is not supported"
trainer = trainer_init(config)
if run_type == "train":
trainer.train()
elif run_type == "eval":
trainer.eval()
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--input-type",
default="blind",
choices=["blind", "rgb", "depth", "rgbd"],
)
parser.add_argument("--evaluation", type=str, required=True, choices=["local", "remote"])
config_paths = os.environ["CHALLENGE_CONFIG_FILE"]
parser.add_argument("--model-path", default="", type=str)
args = parser.parse_args()
config = get_config('configs/ddppo_pointnav.yaml',
['BASE_TASK_CONFIG_PATH', config_paths]).clone()
config.defrost()
config.TORCH_GPU_ID = 0
config.INPUT_TYPE = args.input_type
config.MODEL_PATH = args.model_path
config.RANDOM_SEED = 7
config.freeze()
agent = DDPPOAgent(config)
if args.evaluation == "local":
challenge = habitat.Challenge(eval_remote=False)
else:
challenge = habitat.Challenge(eval_remote=True)
challenge.submit(agent)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--run-type",
choices=["train", "eval"],
required=True,
help="run type of the experiment (train or eval)",
)
parser.add_argument(
"--exp-config",
type=str,
required=True,
help="path to config yaml containing info about experiment",
)
parser.add_argument(
"opts",
default=None,
nargs=argparse.REMAINDER,
help="Modify config options from command line",
)
args = parser.parse_args()
config = get_config(args.exp_config, args.opts)
random.seed(config.TASK_CONFIG.SEED)
np.random.seed(config.TASK_CONFIG.SEED)
trainer_init = baseline_registry.get_trainer(config.TRAINER_NAME)
assert trainer_init is not None, f"{config.TRAINER_NAME} is not supported"
trainer = trainer_init(config)
if args.run_type == "train":
trainer.train()
elif args.run_type == "eval":
trainer.eval()
def run_exp(exp_config: str, run_type: str, opts=None) -> None:
r"""Runs experiment given mode and config
Args:
exp_config: path to config file.
run_type: "train" or "eval.
opts: list of strings of additional config options.
Returns:
None.
"""
config = get_config(exp_config, opts)
config.defrost()
config.TASK_CONFIG.TASK.POINTGOAL_WITH_GPS_SENSOR = config.TASK_CONFIG.TASK.POINTGOAL_SENSOR.clone(
)
config.TASK_CONFIG.TASK.POINTGOAL_WITH_GPS_SENSOR.TYPE = (
"PointGoalWithGPSSensor")
config.freeze()
random.seed(config.TASK_CONFIG.SEED)
np.random.seed(config.TASK_CONFIG.SEED)
torch.manual_seed(config.TASK_CONFIG.SEED)
if config.FORCE_TORCH_SINGLE_THREADED and torch.cuda.is_available():
torch.set_num_threads(1)
trainer_init = baseline_registry.get_trainer(config.TRAINER_NAME)
assert trainer_init is not None, f"{config.TRAINER_NAME} is not supported"
trainer = trainer_init(config)
if run_type == "train":
trainer.train()
elif run_type == "eval":
trainer.eval()
def make_orb_config():
baseline_config = get_config("configs/tasks/pointnav_rgbd.yaml")
config = habitat.get_config("configs/tasks/pointnav_rgbd.yaml")
config.defrost()
baseline_config.defrost()
config.TASK_CONFIG = baseline_config.TASK_CONFIG
config.ORBSLAM2 = baseline_config.ORBSLAM2
config.ORBSLAM2.SLAM_VOCAB_PATH = "data/ORBvoc.txt"
config.ORBSLAM2.SLAM_SETTINGS_PATH = "configs/orbslam2/mp3d3_small1k.yaml"
config.ORBSLAM2.DIST_TO_STOP = 0.2
config.ORBSLAM2.MAP_CELL_SIZE = 0.10
config.SIMULATOR.AGENT_0.SENSORS = [
"RGB_SENSOR",
"DEPTH_SENSOR",
]
config.SIMULATOR.RGB_SENSOR.WIDTH = 256
config.SIMULATOR.RGB_SENSOR.HEIGHT = 256
config.SIMULATOR.DEPTH_SENSOR.WIDTH = 256
config.SIMULATOR.DEPTH_SENSOR.HEIGHT = 256
config.ORBSLAM2.CAMERA_HEIGHT = config.SIMULATOR.DEPTH_SENSOR.POSITION[1]
config.ORBSLAM2.H_OBSTACLE_MIN = 0.3 * config.ORBSLAM2.CAMERA_HEIGHT
config.ORBSLAM2.H_OBSTACLE_MAX = 1.0 * config.ORBSLAM2.CAMERA_HEIGHT
config.ORBSLAM2.MIN_PTS_IN_OBSTACLE = config.SIMULATOR.DEPTH_SENSOR.WIDTH / 2.0
config.freeze()
baseline_config.freeze()
return config
def train_agent(agent):
r"""Runs experiment given mode and config
Args:
exp_config: path to config file.
run_type: "train" or "eval.
opts: list of strings of additional config options.
Returns:
None.
"""
config_file = r"/home/userone/workspace/bed1/ppo_custom/ppo_pointnav_example.yaml"
config = get_config(config_file, None)
# config = get_config1(agent)
env = construct_envs(config, get_env_class(config.ENV_NAME))
config.defrost()
config.TASK_CONFIG.DATASET.DATA_PATH = config.DATA_PATH_SET
config.TASK_CONFIG.DATASET.SCENES_DIR = config.SCENE_DIR_SET
config.freeze()
random.seed(config.TASK_CONFIG.SEED)
np.random.seed(config.TASK_CONFIG.SEED)
if agent == "ppo_agent":
trainer = ppo_trainer.PPOTrainer(config)
trainer.train(env)
def test_eval_config():
ckpt_opts = ["VIDEO_OPTION", "[]"]
eval_opts = ["VIDEO_OPTION", "['disk']"]
ckpt_cfg = get_config(None, ckpt_opts)
assert ckpt_cfg.VIDEO_OPTION == []
assert ckpt_cfg.CMD_TRAILING_OPTS == ["VIDEO_OPTION", "[]"]
eval_cfg = get_config(None, eval_opts)
assert eval_cfg.VIDEO_OPTION == ["disk"]
assert eval_cfg.CMD_TRAILING_OPTS == ["VIDEO_OPTION", "['disk']"]
trainer = BaseRLTrainer(get_config())
assert trainer.config.VIDEO_OPTION == ["disk", "tensorboard"]
returned_config = trainer._setup_eval_config(checkpoint_config=ckpt_cfg)
assert returned_config.VIDEO_OPTION == []
trainer = BaseRLTrainer(eval_cfg)
returned_config = trainer._setup_eval_config(ckpt_cfg)
assert returned_config.VIDEO_OPTION == ["disk"]
def test_cubemap_stiching(test_cfg_path: str, mode: str, camera: str):
meta_config = get_config(config_paths=test_cfg_path)
meta_config.defrost()
config = meta_config.TASK_CONFIG
CAMERA_NUM = 6
orient = [
[0, math.pi, 0], # Back
[-math.pi / 2, 0, 0], # Down
[0, 0, 0], # Front
[0, math.pi / 2, 0], # Right
[0, 3 / 2 * math.pi, 0], # Left
[math.pi / 2, 0, 0], # Up
]
sensor_uuids = []
if "RGB_SENSOR" in config.SIMULATOR.AGENT_0.SENSORS:
config.SIMULATOR.RGB_SENSOR.ORIENTATION = orient[0]
for camera_id in range(1, CAMERA_NUM):
camera_template = f"RGB_{camera_id}"
camera_config = deepcopy(config.SIMULATOR.RGB_SENSOR)
camera_config.ORIENTATION = orient[camera_id]
camera_config.UUID = camera_template.lower()
sensor_uuids.append(camera_config.UUID)
setattr(config.SIMULATOR, camera_template, camera_config)
config.SIMULATOR.AGENT_0.SENSORS.append(camera_template)
if "DEPTH_SENSOR" in config.SIMULATOR.AGENT_0.SENSORS:
config.SIMULATOR.DEPTH_SENSOR.ORIENTATION = orient[0]
for camera_id in range(1, CAMERA_NUM):
camera_template = f"DEPTH_{camera_id}"
camera_config = deepcopy(config.SIMULATOR.DEPTH_SENSOR)
camera_config.ORIENTATION = orient[camera_id]
camera_config.UUID = camera_template.lower()
sensor_uuids.append(camera_config.UUID)
setattr(config.SIMULATOR, camera_template, camera_config)
config.SIMULATOR.AGENT_0.SENSORS.append(camera_template)
meta_config.TASK_CONFIG = config
meta_config.SENSORS = config.SIMULATOR.AGENT_0.SENSORS
if camera == "equirec":
meta_config.RL.POLICY.OBS_TRANSFORMS.CUBE2EQ.SENSOR_UUIDS = tuple(
sensor_uuids)
elif camera == "fisheye":
meta_config.RL.POLICY.OBS_TRANSFORMS.CUBE2FISH.SENSOR_UUIDS = tuple(
sensor_uuids)
meta_config.freeze()
execute_exp(meta_config, mode)
# Deinit processes group
if torch.distributed.is_initialized():
torch.distributed.destroy_process_group()
def run_exp(exp_config: str, run_type: str, opts=None) -> None:
r"""Runs experiment given mode and config
Args:
exp_config: path to config file.
run_type: "train" or "eval.
opts: list of strings of additional config options.
Returns:
None.
"""
config = get_config(exp_config, opts)
execute_exp(config, run_type)
def run_random_agent():
config_file = r"/home/userone/workspace/bed1/ppo_custom/ppo_pointnav_example.yaml"
config = get_config(config_file, None)
# config = get_config1(agent)
env = construct_envs(config, get_env_class(config.ENV_NAME))
config.defrost()
config.TASK_CONFIG.DATASET.DATA_PATH = config.DATA_PATH_SET
config.TASK_CONFIG.DATASET.SCENES_DIR = config.SCENE_DIR_SET
config.freeze()
random.seed(config.TASK_CONFIG.SEED)
np.random.seed(config.TASK_CONFIG.SEED)
random_agent.run(config, env, 3)
def threaded_vector_example():
"""
Initialises a `habitat.ThreadedVectorEnv` environment, then takes random steps.
"""
baseline_config = get_config("configs/tasks/pointnav.yaml")
config = habitat.get_config("configs/tasks/pointnav.yaml")
config.defrost()
config.TASK_CONFIG = baseline_config.TASK_CONFIG
config.NUM_ENVIRONMENTS = 14
config.SENSORS = baseline_config.SENSORS
config.SIMULATOR_GPU_ID = 0
config.freeze()
env_class = habitat.Env
envs = construct_threaded_envs(config, env_class)
_ = envs.reset()
n_steps = 500
n_envs = envs.num_envs
n_episodes = envs.number_of_episodes[0]
for episode in range(n_episodes):
count_steps = 0
print(
f"Starting episode {episode}/{n_episodes} in parallel threads across {n_envs} environments."
)
scene_names = [
ep.scene_id.split("/")[-1].strip(".glb")
for ep in envs.current_episodes()
]
for i, scene in enumerate(scene_names):
print(f"Stepping around in Environment {i+1}: {scene}..")
for step in range(n_steps):
random_actions = [
action_space.sample() for action_space in envs.action_spaces
]
_ = envs.step(random_actions)
count_steps += 1
print("Episodes finished after {} steps.".format(count_steps))
def trigger_transfer_learn(agent):
config_file = r"/home/userone/workspace/bed1/ppo_custom/ppo_pointnav_example.yaml"
config = get_config(config_file, None)
# config = get_config1(agent)
env = construct_envs(config, get_env_class(config.ENV_NAME))
config.defrost()
config.TASK_CONFIG.DATASET.DATA_PATH = config.DATA_PATH_SET
config.TASK_CONFIG.DATASET.SCENES_DIR = config.SCENE_DIR_SET
config.freeze()
random.seed(config.TASK_CONFIG.SEED)
np.random.seed(config.TASK_CONFIG.SEED)
if agent == "ppo_agent":
trainer = transfer_ppo.PPOTrainer(config)
trainer.train(env)
def run_exp(exp_config: str, run_type: str, opts=None) -> None:
r"""Runs experiment given mode and config
Args:
exp_config: path to config file.
run_type: "train" or "eval.
opts: list of strings of additional config options.
Returns:
None.
"""
if 'RL.POLICY.fuse_states' in opts:
i = opts.index('RL.POLICY.fuse_states')
opts[i + 1] = opts[i + 1].split(',')
if len(opts[i + 1]) == 1 and opts[i + 1][0] == '':
opts[i + 1] = []
config = get_config(exp_config, opts)
execute_exp(config, run_type)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--task-config",
type=str,
default="configs/tasks/vln_r2r.yaml")
parser.add_argument("--num-episodes", type=int, default=100)
parser.add_argument("--agent_type", type=int, default=0)
parser.add_argument("--discrete", action='store_true')
parser.add_argument("--train", action='store_true')
parser.add_argument("--val", action='store_true')
parser.add_argument("--feedback", type=int, default=0)
args = parser.parse_args()
feedback_options = ["teacher", "argmax", "sample"]
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
#print(device)
#encoder = EncoderLSTM(1300, 256, 256, 0, 0.5, bidirectional=False, num_layers=2).to(device)
#decoder = AttnDecoderLSTM(8, 6, 32, 256, 0.5).to(device)
#agent = seq2seqAgent(3.0, "SPL", encoder, decoder)
#benchmark = Seq2SeqBenchmark(args.task_config)
cfg = get_config(args.task_config)
if args.train:
assert 0 <= args.feedback <= 2, "Incorrect feedback option"
print("Running training with feedback %s" %
feedback_options[args.feedback])
trainer = vln_trainer.VLNTrainer(cfg)
trainer.train()
#metrics = benchmark.train(agent, num_episodes=args.num_episodes, feedback=feedback_options[args.feedback])
#for k, v in metrics.items():
# print("{0}: {1}".format(k, v))
if args.val:
count_episodes = 5001
agent.load("checkpoints/encoder_train_{}.check".format(count_episodes),
"checkpoints/decoder_train_{}.check".format(count_episodes))
metrics = benchmark.evaluate(agent, num_episodes=args.num_episodes)
for k, v in metrics.items():
print("{0}: {1}".format(k, v))
def example():
"""
Initialises a `habitat.Env` environment, then takes random steps.
"""
baseline_config = get_config("configs/tasks/pointnav.yaml")
config = habitat.get_config("configs/tasks/pointnav.yaml")
config.defrost()
config.TASK_CONFIG = baseline_config.TASK_CONFIG
config.freeze()
env_class = habitat.Env
env = make_env_fn(config, env_class)
n_steps = 500
n_episodes = env.number_of_episodes
for episode in range(n_steps):
count_steps = 0
_ = env.reset()
scene_name = env.current_episode.scene_id.split(
sep="/")[-1].strip(".glb")
print(
f"\nStarting episode {episode+1}/{n_episodes} in {scene_name}...")
print("Stepping around...")
for step in range(n_steps):
action = env.action_space.sample()
while action["action"] == "STOP":
action = env.action_space.sample()
_ = env.step(action)
count_steps += 1
print(
f"Episode {episode+1} in scene {scene_name} finished after {count_steps} steps."
)
def run_exp(exp_config: str,
run_type: str,
ckpt_path="",
run_id=None,
run_suffix="",
opts=None) -> None:
r"""Runs experiment given mode and config
Args:
exp_config: path to config file.
run_type: "train" or "eval.
ckpt_path: If training, ckpt to resume. If evaluating, ckpt to evaluate.
run_id: If using slurm batch, run id to prefix.s
opts: list of strings of additional config options.
Returns:
None.
"""
config = get_config(exp_config, opts)
# Add tracking of the number of episodes
config.defrost()
config.TASK_CONFIG.TASK.EPISODE_INFO_EXAMPLE = habitat.Config()
# The type field is used to look-up the measure in the registry.
# By default, the things are registered with the class name
config.TASK_CONFIG.TASK.EPISODE_INFO_EXAMPLE.TYPE = "EpisodeInfoExample"
config.TASK_CONFIG.TASK.EPISODE_INFO_EXAMPLE.VALUE = 5
# Add the measure to the list of measures in use
config.TASK_CONFIG.TASK.MEASUREMENTS.append("EPISODE_INFO_EXAMPLE")
config.freeze()
variant_name = os.path.split(exp_config)[1].split('.')[0]
config.defrost()
if run_suffix != "" and run_suffix is not None:
variant_name = f"{variant_name}-{run_suffix}"
if not osp.exists(config.LOG_FILE):
os.makedirs(config.LOG_FILE)
config.TENSORBOARD_DIR = os.path.join(config.TENSORBOARD_DIR, variant_name)
config.CHECKPOINT_FOLDER = os.path.join(config.CHECKPOINT_FOLDER,
variant_name)
config.VIDEO_DIR = os.path.join(config.VIDEO_DIR, variant_name)
config.LOG_FILE = os.path.join(config.LOG_FILE,
f"{variant_name}.log") # actually a logdir
if run_type == "eval":
# config.TRAINER_NAME = "ppo"
config.NUM_PROCESSES = 6 # nice
else:
# Add necessary supervisory signals
train_sensors = config.RL.AUX_TASKS.required_sensors
config.SENSORS.extend(
train_sensors) # the task cfg sensors are overwritten by this one
if run_id is None:
random.seed(config.TASK_CONFIG.SEED)
np.random.seed(config.TASK_CONFIG.SEED)
trainer_init = baseline_registry.get_trainer(config.TRAINER_NAME)
assert trainer_init is not None, f"{config.TRAINER_NAME} is not supported"
trainer = trainer_init(config)
# If not doing multiple runs (with run_id), default behavior is to overwrite
if run_type == "train":
if ckpt_path is not None:
ckpt_dir, ckpt_file = os.path.split(ckpt_path)
ckpt_index = ckpt_file.split('.')[1]
ckpt = int(ckpt_index)
start_updates = ckpt * config.CHECKPOINT_INTERVAL + 1
trainer.train(ckpt_path=ckpt_path,
ckpt=ckpt,
start_updates=start_updates)
elif not DO_PRESERVE_RUNS:
# if os.path.exists(config.TENSORBOARD_DIR):
# print("Removing tensorboard directory...")
# shutil.rmtree(config.TENSORBOARD_DIR, ignore_errors=True)
# if os.path.exists(config.CHECKPOINT_FOLDER):
# print("Removing checkpoint folder...")
# shutil.rmtree(config.CHECKPOINT_FOLDER, ignore_errors=True)
# if os.path.exists(config.LOG_FILE):
# print("Removing log file...")
# shutil.rmtree(config.LOG_FILE, ignore_errors=True)
trainer.train()
else:
# if os.path.exists(config.TENSORBOARD_DIR) or os.path.exists(config.CHECKPOINT_FOLDER) \
# or os.path.exists(config.LOG_FILE):
# print(f"TB dir exists: {os.path.exists(config.TENSORBOARD_DIR)}")
# print(f"Ckpt dir exists: {os.path.exists(config.CHECKPOINT_FOLDER)}")
# print(f"Log file exists: {os.path.exists(config.LOG_FILE)}")
# print("Run artifact exists, please clear manually")
# exit(1)
trainer.train()
elif run_type == "eval":
trainer.eval(ckpt_path)
return
run_prefix = f'run_{run_id}'
seed = run_id
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(config.TASK_CONFIG.SEED)
# Exetnds off old modifications
tb_dir = os.path.join(config.TENSORBOARD_DIR, run_prefix)
ckpt_dir = os.path.join(config.CHECKPOINT_FOLDER, run_prefix)
log_dir, log_file = os.path.split(config.LOG_FILE)
log_file_extended = f"{run_prefix}--{log_file}"
log_file_path = os.path.join(log_dir, log_file_extended)
config.TASK_CONFIG.SEED = seed
config.TENSORBOARD_DIR = tb_dir
config.CHECKPOINT_FOLDER = ckpt_dir
config.LOG_FILE = log_file_path
trainer_init = baseline_registry.get_trainer(config.TRAINER_NAME)
assert trainer_init is not None, f"{config.TRAINER_NAME} is not supported"
trainer = trainer_init(config)
if run_type == "train":
if ckpt_path is None:
# if DO_PRESERVE_RUNS and (os.path.exists(tb_dir) or os.path.exists(ckpt_dir) or os.path.exists(log_file_extended)):
# print(f"TB dir exists: {os.path.exists(tb_dir)}")
# print(f"Ckpt dir exists: {os.path.exists(ckpt_dir)}")
# print(f"Log file exists: {os.path.exists(log_file_extended)}")
# print("Run artifact exists, please clear manually")
# exit(1)
# else:
# shutil.rmtree(tb_dir, ignore_errors=True)
# shutil.rmtree(ckpt_dir, ignore_errors=True)
# if os.path.exists(log_file_extended):
# os.remove(log_file_extended)
trainer.train()
else: # Resume training from checkpoint
# Parse the checkpoint #, calculate num updates, update the config
ckpt_dir, ckpt_file = os.path.split(ckpt_path)
ckpt_index = ckpt_file.split('.')[1]
true_path = os.path.join(ckpt_dir, run_prefix,
f"{run_prefix}.{ckpt_index}.pth")
ckpt = int(ckpt_index)
start_updates = ckpt * config.CHECKPOINT_INTERVAL + 1
trainer.train(ckpt_path=true_path,
ckpt=ckpt,
start_updates=start_updates)
else:
ckpt_dir, ckpt_file = os.path.split(ckpt_path)
ckpt_index = ckpt_file.split('.')[1]
true_path = os.path.join(ckpt_dir, run_prefix,
f"{run_prefix}.{ckpt_index}.pth")
trainer.eval(true_path)
def test_cubemap_stiching(test_cfg_path: str, mode: str, camera: str,
sensor_type: str):
meta_config = get_config(config_paths=test_cfg_path)
meta_config.defrost()
config = meta_config.TASK_CONFIG
CAMERA_NUM = 6
orient = [
[0, math.pi, 0], # Back
[-math.pi / 2, 0, 0], # Down
[0, 0, 0], # Front
[0, math.pi / 2, 0], # Right
[0, 3 / 2 * math.pi, 0], # Left
[math.pi / 2, 0, 0], # Up
]
sensor_uuids = []
if f"{sensor_type}_SENSOR" not in config.SIMULATOR.AGENT_0.SENSORS:
config.SIMULATOR.AGENT_0.SENSORS.append(f"{sensor_type}_SENSOR")
sensor = getattr(config.SIMULATOR, f"{sensor_type}_SENSOR")
for camera_id in range(CAMERA_NUM):
camera_template = f"{sensor_type}_{camera_id}"
camera_config = deepcopy(sensor)
camera_config.ORIENTATION = orient[camera_id]
camera_config.UUID = camera_template.lower()
sensor_uuids.append(camera_config.UUID)
setattr(config.SIMULATOR, camera_template, camera_config)
config.SIMULATOR.AGENT_0.SENSORS.append(camera_template)
meta_config.TASK_CONFIG = config
meta_config.SENSORS = config.SIMULATOR.AGENT_0.SENSORS
if camera == "equirect":
meta_config.RL.POLICY.OBS_TRANSFORMS.CUBE2EQ.SENSOR_UUIDS = tuple(
sensor_uuids)
elif camera == "fisheye":
meta_config.RL.POLICY.OBS_TRANSFORMS.CUBE2FISH.SENSOR_UUIDS = tuple(
sensor_uuids)
meta_config.freeze()
if camera in ["equirect", "fisheye"]:
execute_exp(meta_config, mode)
# Deinit processes group
if torch.distributed.is_initialized():
torch.distributed.destroy_process_group()
elif camera == "cubemap":
# 1) Generate an equirect image from cubemap images.
# 2) Generate cubemap images from the equirect image.
# 3) Compare the input and output cubemap
env_fn_args = []
for split in ["train", "val"]:
tmp_config = config.clone()
tmp_config.defrost()
tmp_config.DATASET["SPLIT"] = split
tmp_config.freeze()
env_fn_args.append((tmp_config, None))
with VectorEnv(env_fn_args=env_fn_args) as envs:
observations = envs.reset()
batch = batch_obs(observations)
orig_batch = deepcopy(batch)
# ProjectionTransformer
obs_trans_to_eq = baseline_registry.get_obs_transformer(
"CubeMap2Equirect")
cube2equirect = obs_trans_to_eq(sensor_uuids, (256, 512))
obs_trans_to_cube = baseline_registry.get_obs_transformer(
"Equirect2CubeMap")
equirect2cube = obs_trans_to_cube(cube2equirect.target_uuids,
(256, 256))
# Cubemap to Equirect to Cubemap
batch_eq = cube2equirect(batch)
batch_cube = equirect2cube(batch_eq)
# Extract input and output cubemap
output_cube = batch_cube[cube2equirect.target_uuids[0]]
input_cube = [orig_batch[key] for key in sensor_uuids]
input_cube = torch.stack(input_cube, axis=1)
input_cube = torch.flatten(input_cube, end_dim=1)
# Apply blur to absorb difference (blur, etc.) caused by conversion
if sensor_type == "RGB":
output_cube = output_cube.float() / 255
input_cube = input_cube.float() / 255
output_cube = output_cube.permute((0, 3, 1, 2)) # NHWC => NCHW
input_cube = input_cube.permute((0, 3, 1, 2)) # NHWC => NCHW
apply_blur = torch.nn.AvgPool2d(5, 3, 2)
output_cube = apply_blur(output_cube)
input_cube = apply_blur(input_cube)
# Calculate the difference
diff = torch.abs(output_cube - input_cube)
assert diff.mean().item() < 0.01
else:
raise ValueError(f"Unknown camera name: {camera}")
def _worker_fn(world_rank: int, world_size: int, port: int,
unused_params: bool):
device = (torch.device("cuda")
if torch.cuda.is_available() else torch.device("cpu"))
tcp_store = distrib.TCPStore( # type: ignore
"127.0.0.1", port, world_size, world_rank == 0)
distrib.init_process_group("gloo",
store=tcp_store,
rank=world_rank,
world_size=world_size)
config = get_config("habitat_baselines/config/test/ppo_pointnav_test.yaml")
obs_space = gym.spaces.Dict({
IntegratedPointGoalGPSAndCompassSensor.cls_uuid:
gym.spaces.Box(
low=np.finfo(np.float32).min,
high=np.finfo(np.float32).max,
shape=(2, ),
dtype=np.float32,
)
})
action_space = ActionSpace({"move": EmptySpace()})
actor_critic = PointNavBaselinePolicy.from_config(config, obs_space,
action_space)
# This use adds some arbitrary parameters that aren't part of the computation
# graph, so they will mess up DDP if they aren't correctly ignored by it
if unused_params:
actor_critic.unused = nn.Linear(64, 64)
actor_critic.to(device=device)
ppo_cfg = config.RL.PPO
agent = DDPPO(
actor_critic=actor_critic,
clip_param=ppo_cfg.clip_param,
ppo_epoch=ppo_cfg.ppo_epoch,
num_mini_batch=ppo_cfg.num_mini_batch,
value_loss_coef=ppo_cfg.value_loss_coef,
entropy_coef=ppo_cfg.entropy_coef,
lr=ppo_cfg.lr,
eps=ppo_cfg.eps,
max_grad_norm=ppo_cfg.max_grad_norm,
use_normalized_advantage=ppo_cfg.use_normalized_advantage,
)
agent.init_distributed()
rollouts = RolloutStorage(
ppo_cfg.num_steps,
2,
obs_space,
action_space,
ppo_cfg.hidden_size,
num_recurrent_layers=actor_critic.net.num_recurrent_layers,
is_double_buffered=False,
)
rollouts.to(device)
for k, v in rollouts.buffers["observations"].items():
rollouts.buffers["observations"][k] = torch.randn_like(v)
# Add two steps so batching works
rollouts.advance_rollout()
rollouts.advance_rollout()
# Get a single batch
batch = next(rollouts.recurrent_generator(rollouts.buffers["returns"], 1))
# Call eval actions through the internal wrapper that is used in
# agent.update
value, action_log_probs, dist_entropy, _ = agent._evaluate_actions(
batch["observations"],
batch["recurrent_hidden_states"],
batch["prev_actions"],
batch["masks"],
batch["actions"],
)
# Backprop on things
(value.mean() + action_log_probs.mean() + dist_entropy.mean()).backward()
# Make sure all ranks have very similar parameters
for param in actor_critic.parameters():
if param.grad is not None:
grads = [param.grad.detach().clone() for _ in range(world_size)]
distrib.all_gather(grads, grads[world_rank])
for i in range(world_size):
assert torch.isclose(grads[i], grads[world_rank]).all()
from habitat_baselines.config.default import get_config
from habitat_baselines.common.pepper_env import PepperRLExplorationEnv
import cv2
import pickle
import numpy as np
import matplotlib.pyplot as plt
rgb_buffer = []
depth_buffer = []
forward_step = 0.25
turn_step = 0.1
cfg = get_config()
print(cfg)
pepper_env = PepperRLExplorationEnv(cfg)
pepper_env.reset()
key = 0
c_action = np.random.choice(3, 1, p=[0.8, 0.1, 0.1])[0]
default_rot = np.random.choice(3, 1, p=[0, 0.5, 0.5])[0]
values = []
forward_enabled = True
user_forward_enabled = True
num_forward = 0
actions = []
step = 0
observations, reward, done, info = \
pepper_env.reset()
last_pose = pepper_env.get_position()[0]
def run_exp(exp_config: str, run_type: str, opts=None,
experiment: str = None, gpu_id: int = 0, other_patterns=None) \
-> None:
r"""Runs experiment given mode and config
Args:
exp_config: path to config file.
run_type: "train" or "eval.
opts: list of strings of additional config options.
Returns:
None.
"""
if run_type == "train":
replace_config = dict()
# Change config based on arguments
commit_hash = get_git_revision_hash()
replace_config[CFG_COMMIT_HASH] = commit_hash
# New folder path based on timestamp & prefix
fld_name = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
if experiment is not None:
fld_name += f"_{experiment}"
replace_config[CFG_RESULTS_PREFIX] = fld_name
replace_config[CFG_GPU] = gpu_id
# Add other regex patterns to change in config
if other_patterns is not None:
assert len(other_patterns) % 2 == 0, "Must list pairs of arguments"
for k, v in zip(other_patterns[::2], other_patterns[1::2]):
replace_config[k] = v
with open(exp_config, "r") as f:
config_file_lines = f.readlines()
config_file_lines = preprocess_config(config_file_lines,
replace_config)
path = None
# Get out folder path
for line in config_file_lines:
if line.startswith(RESULTS_FOLDER_PATH):
path = yaml.load(line)[RESULTS_FOLDER_PATH]
path = path[:path.find(fld_name) + len(fld_name)]
break
assert path is not None, "Results path not found"
# Generate folder for results
os.makedirs(path)
cfg_name = os.path.basename(exp_config)
new_cfg = os.path.join(path, cfg_name)
with open(new_cfg, "w") as f:
f.writelines(config_file_lines)
# Read new generated config
exp_config = new_cfg
config = get_config(exp_config, opts)
# Random seed
random.seed(config.TASK_CONFIG.SEED)
torch.manual_seed(config.TASK_CONFIG.SEED)
np.random.seed(config.TASK_CONFIG.SEED)
torch.backends.cudnn.deterministic = True # Slower that normal
torch.backends.cudnn.benchmark = False
trainer_init = baseline_registry.get_trainer(config.TRAINER_NAME)
assert trainer_init is not None, f"{config.TRAINER_NAME} is not supported"
trainer = trainer_init(config)
if run_type == "train":
trainer.train()
elif run_type == "eval":
trainer.eval()
def run_exp(exp_config: str, run_type: str, ckpt_path="", run_id=None, run_suffix=None, opts=None) -> None:
r"""Runs experiment given mode and config
Args:
exp_config: path to config file.
run_type: "train" or "eval.
ckpt_path: If evaluating, path to a checkpoint.
run_id: If using slurm batch, run id to prefix.
opts: list of strings of additional config options.
Returns:
None.
"""
if run_type != "eval":
print("Detailed runs only supported for evaluation")
exit(1)
config = get_config(exp_config, opts)
variant_name = os.path.split(exp_config)[1].split('.')[0]
config.defrost()
if run_suffix != "" and run_suffix is not None:
variant_name = f"{variant_name}-{run_suffix}"
config.TENSORBOARD_DIR = os.path.join(config.TENSORBOARD_DIR, variant_name)
config.CHECKPOINT_FOLDER = os.path.join(config.CHECKPOINT_FOLDER, variant_name)
config.LOG_FILE = os.path.join(config.LOG_FILE, f"{variant_name}.log") # actually a logdir
config.NUM_PROCESSES = 1
run_prefix = 'run'
if run_id is not None:
config.TASK_CONFIG.SEED = run_id
run_prefix = f'run_{run_id}'
random.seed(config.TASK_CONFIG.SEED)
np.random.seed(config.TASK_CONFIG.SEED)
# Sample input - /baseline/run_0.80.pth - need to add the extra folder
ckpt_dir, ckpt_file = os.path.split(ckpt_path)
variant = ckpt_dir.split('/')[-1]
ckpt_index = ckpt_file.split('.')[1]
ckpt_path = os.path.join(ckpt_dir, run_prefix, ckpt_file)
# This config isn't used for detailed statistics, just put there somewhere where they won't overwrite
# * Modify as desired
detail_dir = os.path.join("/srv/share/jye72/r2_viz", "map_viz", f"{variant}_{run_prefix}_{ckpt_index}")
config.VIDEO_DIR = os.path.join(detail_dir)
# * Modify
make_background = False
map_name = "quantico"
eval_stats_dir = os.path.join(f'/nethome/jye72/share/r3_detailed/') # /{map_name}')
config.TASK_CONFIG.TASK_SENSORS = [
'POINTGOAL_WITH_GPS_COMPASS_SENSOR',
'GPS_SENSOR', 'HEADING_SENSOR'
]
if make_background:
config.TEST_EPISODE_COUNT = 1
label = f"{variant}_{ckpt_index}_{run_prefix}-bg"
log_diagnostics = [Diagnostics.basic, Diagnostics.top_down_map]
else:
config.VIDEO_OPTION = []
label = f"{variant}_{ckpt_index}_{run_prefix}"
# log_diagnostics = [Diagnostics.basic, Diagnostics.actions,
# Diagnostics.weights, Diagnostics.gps, Diagnostics.heading]
log_diagnostics = [Diagnostics.basic] # , Diagnostics.actions, Diagnostics.weights]
log_diagnostics = [Diagnostics.basic, Diagnostics.actions, Diagnostics.weights]
# * Modify as desired
use_own_dataset = False
if use_own_dataset:
config.TASK_CONFIG.DATASET.DATA_PATH = \
'/nethome/jye72/projects/data/datasets/pointnav/gibson/scene_viz/{split}/{split}.json.gz'
config.TASK_CONFIG.DATASET.SPLIT = 'all'
config.TASK_CONFIG.DATASET.SCENES_DIR = "data/scene_datasets/gibson"
config.TASK_CONFIG.EVAL.SPLIT = 'all' # quirk in the code necessitates this
# Make a top-down-map clean for visualization
map_cfg = config.TASK_CONFIG.TASK.TOP_DOWN_MAP
map_cfg.MAP_RESOLUTION = 4000
map_cfg.DRAW_SOURCE = False
map_cfg.DRAW_GOAL_POSITIONS = False
map_cfg.DRAW_VIEW_POINTS = False
map_cfg.DRAW_SHORTEST_PATH = False
trainer_init = baseline_registry.get_trainer(config.TRAINER_NAME)
assert trainer_init is not None, f"{config.TRAINER_NAME} is not supported"
trainer = trainer_init(config)
trainer.eval(ckpt_path, log_diagnostics=log_diagnostics,
output_dir=eval_stats_dir, label=label)
def __init__(self,
wrapper,
config_file,
base_config_file=None,
horizon=None,
gamma=0.99,
width=None,
height=None):
"""
Constructor. For more details on how to pass YAML configuration files,
please see <MUSHROOM_RL PATH>/examples/habitat/README.md
Args:
wrapper (str): wrapper for converting observations and actions
(e.g., HabitatRearrangeWrapper);
config_file (str): path to the YAML file specifying the RL task
configuration (see <HABITAT_LAB PATH>/habitat_baselines/configs/);
base_config_file (str, None): path to an optional YAML file, used
as 'BASE_TASK_CONFIG_PATH' in the first YAML
(see <HABITAT_LAB PATH>/configs/);
horizon (int, None): the horizon;
gamma (float, 0.99): the discount factor;
width (int, None): width of the pixel observation. If None, the
value specified in the config file is used.
height (int, None): height of the pixel observation. If None, the
value specified in the config file is used.
"""
# MDP creation
self._not_pybullet = False
self._first = True
if base_config_file is None:
base_config_file = config_file
config = get_config(config_paths=config_file,
opts=['BASE_TASK_CONFIG_PATH', base_config_file])
config.defrost()
if horizon is None:
horizon = config.TASK_CONFIG.ENVIRONMENT.MAX_EPISODE_STEPS # Get the default horizon
config.TASK_CONFIG.ENVIRONMENT.MAX_EPISODE_STEPS = horizon + 1 # Hack to ignore gym time limit
# Overwrite all RGB width / height used for the TASK (not SIMULATOR)
for k in config['TASK_CONFIG']['SIMULATOR']:
if 'rgb' in k.lower():
if height is not None:
config['TASK_CONFIG']['SIMULATOR'][k]['HEIGHT'] = height
if width is not None:
config['TASK_CONFIG']['SIMULATOR'][k]['WIDTH'] = width
config.freeze()
env_class = get_env_class(config.ENV_NAME)
env = make_env_fn(env_class=env_class, config=config)
env = globals()[wrapper](env)
self.env = env
self._img_size = env.observation_space.shape[0:2]
# MDP properties
action_space = self.env.action_space
observation_space = Box(low=0.,
high=255.,
shape=(3, self._img_size[1],
self._img_size[0]))
mdp_info = MDPInfo(observation_space, action_space, gamma, horizon)
if isinstance(action_space, Discrete):
self._convert_action = lambda a: a[0]
else:
self._convert_action = lambda a: a
self._viewer = ImageViewer((self._img_size[1], self._img_size[0]),
1 / 10)
Environment.__init__(self, mdp_info)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--skill-type", default="pick")
parser.add_argument("--num-eval", type=int, default=None)
parser.add_argument("--traj-save-path", type=str, default=None)
parser.add_argument(
"--task-cfg",
type=str,
default="habitat_baselines/config/rearrange/spap_rearrangepick.yaml",
)
parser.add_argument(
"opts",
default=None,
nargs=argparse.REMAINDER,
help="Modify config options from command line",
)
args = parser.parse_args()
config = get_config(args.task_cfg, args.opts)
benchmark = BenchmarkGym(
config,
config.VIDEO_OPTIONS,
config.VIDEO_DIR,
{config.RL.SUCCESS_MEASURE},
args.traj_save_path,
should_save_fn=lambda metrics: metrics[config.RL.SUCCESS_MEASURE],
)
ac_cfg = config.TASK_CONFIG.TASK.ACTIONS
spa_cfg = config.SENSE_PLAN_ACT
env = benchmark._env
def get_object_args(skill):
target_idx = skill._sim.get_targets()[0][0]
return {"obj": target_idx}
def get_arm_rest_args(skill):
return {"robot_target": skill._task.desired_resting}
skills = {
"reach":
IkMoveArm(env, spa_cfg, ac_cfg, auto_get_args_fn=get_arm_rest_args),
"pick":
AgentComposition(
[
SpaManipPick(
env, spa_cfg, ac_cfg, auto_get_args_fn=get_object_args),
SpaResetModule(
env,
spa_cfg,
ac_cfg,
ignore_first=True,
auto_get_args_fn=get_object_args,
),
],
env,
spa_cfg,
ac_cfg,
auto_get_args_fn=get_object_args,
),
}
use_skill = skills[args.skill_type]
metrics = benchmark.evaluate(use_skill, args.num_eval)
for k, v in metrics.items():
habitat.logger.info("{}: {:.3f}".format(k, v))
from habitat.tasks.nav.shortest_path_follower import ShortestPathFollower
from habitat_baselines.agents.slam_agents import ORBSLAM2Agent
from habitat_baselines.config.default import get_config
from habitat_baselines.utils.env_utils import make_env_fn
def rotate_origin_only(xy, radians):
"""Only rotate a point around the origin (0, 0)."""
x, y = xy
xx = x * math.cos(radians) + y * math.sin(radians)
yy = -x * math.sin(radians) + y * math.cos(radians)
return xx, yy
baseline_config = get_config("configs/tasks/pointnav_rgbd.yaml")
config = habitat.get_config("configs/tasks/pointnav_rgbd.yaml")
config.defrost()
baseline_config.defrost()
config.TASK_CONFIG = baseline_config.TASK_CONFIG
config.ORBSLAM2 = baseline_config.ORBSLAM2
config.ORBSLAM2.DIST_TO_STOP = 0.20
config.SIMULATOR.AGENT_0.SENSORS = [
"RGB_SENSOR",
"DEPTH_SENSOR",
]
config.SIMULATOR.RGB_SENSOR.WIDTH = 256
config.SIMULATOR.RGB_SENSOR.HEIGHT = 256
config.SIMULATOR.DEPTH_SENSOR.WIDTH = 256
def run_exp(exp_config: str, run_type: str, opts=None) -> None:
r"""Runs experiment given mode and config
Args:
exp_config: path to config file.
run_type: "train" or "eval.
opts: list of strings of additional config options.
Returns:
None.
"""
config = get_config(exp_config, opts)
random.seed(config.TASK_CONFIG.SEED)
np.random.seed(config.TASK_CONFIG.SEED)
torch.manual_seed(config.TASK_CONFIG.SEED)
trainer_init = baseline_registry.get_trainer(config.TRAINER_NAME)
assert trainer_init is not None, f"{config.TRAINER_NAME} is not supported"
trainer = trainer_init(config)
if run_type == "train":
trainer.train()
elif run_type == "eval":
# following are modified based on ppo_trainer.py
actor_critic, batch, device, not_done_masks, test_recurrent_hidden_states = trainer.eval_bruce(
)
def transform_callback(data):
nonlocal actor_critic
nonlocal batch
nonlocal not_done_masks
nonlocal test_recurrent_hidden_states
global flag
global t_prev_update
global observation
if flag == 2:
observation["depth"] = np.reshape(data.data[0:-2], (256, 256, 1))
observation["pointgoal_with_gps_compass"] = data.data[-2:]
flag = 1
return
pointgoal_received = data.data[-2:]
translate_amount = 0.25 # meters
rotate_amount = 0.174533 # radians
isrotated = (rotate_amount * 0.95 <=
abs(pointgoal_received[1] -
observation["pointgoal_with_gps_compass"][1]) <=
rotate_amount * 1.05)
istimeup = (time.time() - t_prev_update) >= 4
# print('istranslated is '+ str(istranslated))
# print('isrotated is '+ str(isrotated))
# print('istimeup is '+ str(istimeup))
if isrotated or istimeup:
vel_msg = Twist()
vel_msg.linear.x = 0
vel_msg.linear.y = 0
vel_msg.linear.z = 0
vel_msg.angular.x = 0
vel_msg.angular.y = 0
vel_msg.angular.z = 0
pub_vel.publish(vel_msg)
time.sleep(0.2)
print("entered update step")
# cv2.imshow("Depth", observation['depth'])
# cv2.waitKey(100)
observation["depth"] = np.reshape(data.data[0:-2], (256, 256, 1))
observation["pointgoal_with_gps_compass"] = data.data[-2:]
batch = batch_obs([observation])
for sensor in batch:
batch[sensor] = batch[sensor].to(device)
if flag == 1:
not_done_masks = torch.tensor([0.0],
dtype=torch.float,
device=device)
flag = 0
else:
not_done_masks = torch.tensor([1.0],
dtype=torch.float,
device=device)
_, actions, _, test_recurrent_hidden_states = actor_critic.act(
batch,
test_recurrent_hidden_states,
not_done_masks,
deterministic=True)
action_id = actions.item()
print("observation received to produce action_id is " +
str(observation["pointgoal_with_gps_compass"]))
print("action_id from net is " + str(actions.item()))
t_prev_update = time.time()
vel_msg = Twist()
vel_msg.linear.x = 0
vel_msg.linear.y = 0
vel_msg.linear.z = 0
vel_msg.angular.x = 0
vel_msg.angular.y = 0
vel_msg.angular.z = 0
if action_id == 0:
vel_msg.linear.x = 0.25 / 4
pub_vel.publish(vel_msg)
elif action_id == 1:
vel_msg.angular.z = 10 / 180 * 3.1415926
pub_vel.publish(vel_msg)
elif action_id == 2:
vel_msg.angular.z = -10 / 180 * 3.1415926
pub_vel.publish(vel_msg)
else:
pub_vel.publish(vel_msg)
sub.unregister()
print("NN finished navigation task")
sub = rospy.Subscriber("depth_and_pointgoal",
numpy_msg(Floats),
transform_callback,
queue_size=1)
rospy.spin()
