def main() -> None:
with initialize(config_path="conf"):
cfg = compose(config_name="config", return_hydra_config=True)
assert cfg.config == {"hello": "world"}
assert cfg.hydra.job.name == "main"
with initialize(config_path="conf", job_name="test_job"):
cfg = compose(config_name="config", return_hydra_config=True)
assert cfg.config == {"hello": "world"}
assert cfg.hydra.job.name == "test_job"
abs_config_dir = os.path.abspath("initialization_test_app/conf")
with initialize_config_dir(config_dir=abs_config_dir):
cfg = compose(config_name="config", return_hydra_config=True)
assert cfg.config == {"hello": "world"}
assert cfg.hydra.job.name == "app"
with initialize_config_dir(config_dir=abs_config_dir, job_name="test_job"):
cfg = compose(config_name="config", return_hydra_config=True)
assert cfg.config == {"hello": "world"}
assert cfg.hydra.job.name == "test_job"
# Those tests can only work if the module is installed
if len(sys.argv) > 1 and sys.argv[1] == "module_installed":
with initialize_config_module(config_module="initialization_test_app.conf"):
cfg = compose(config_name="config", return_hydra_config=True)
assert cfg.config == {"hello": "world"}
assert cfg.hydra.job.name == "app"
with initialize_config_module(
config_module="initialization_test_app.conf", job_name="test_job"
):
cfg = compose(config_name="config", return_hydra_config=True)
assert cfg.config == {"hello": "world"}
assert cfg.hydra.job.name == "test_job"
def test_generated_config(self) -> None:
with initialize_config_module(config_module="hydra_app.conf"):
cfg = compose(config_name="config", overrides=["app.user=test_user"])
assert cfg == {
"app": {"user": "******", "num1": 10, "num2": 20},
"db": {"host": "localhost", "port": 3306},
}
def test_initialize_with_module(hydra_restore_singletons: Any) -> None:
with initialize_config_module(
config_module="tests.test_apps.app_with_cfg_groups.conf", job_name="my_pp"
):
assert compose(config_name="config") == {
"optimizer": {"type": "nesterov", "lr": 0.001}
}
def run_maze_job(hydra_overrides: Dict[str, str], config_module: str, config_name: str) -> DictConfig:
"""Runs rollout with the given config overrides using maze_run.
:param hydra_overrides: Config overrides for hydra.
:param config_module: The config module.
:param config_name: The name of the default config.
"""
with initialize_config_module(config_module=config_module):
# Config is relative to a module
# For the HydraConfig init below, we need the hydra key there as well (=> return_hydra_config=True)
cfg = compose(config_name=config_name,
overrides=[key + "=" + str(val) for key, val in hydra_overrides.items()],
return_hydra_config=True)
# Init the HydraConfig: This is when Hydra actually creates the output dir and changes into it
# (otherwise we only have the config object, but not the full run environment)
HydraConfig.instance().set_config(cfg)
# For the rollout itself, the Hydra config should not be there anymore
with open_dict(cfg):
del cfg["hydra"]
# Run the rollout
maze_run(cfg)
return cfg
def check_env_and_model_instantiation(config_module: str, config: str,
overrides: Dict[str, str]) -> None:
"""Check if env instantiation works."""
with initialize_config_module(config_module):
# config is relative to a module
cfg = compose(
config,
overrides=[key + "=" + value for key, value in overrides.items()])
env_factory = EnvFactory(cfg.env,
cfg.wrappers if "wrappers" in cfg else {})
env = env_factory()
assert env is not None
assert isinstance(env, (StructuredEnv, StructuredEnvSpacesMixin))
if 'model' in overrides and overrides['model'] == 'rllib':
return
if 'model' in cfg:
model_composer = Factory(BaseModelComposer).instantiate(
cfg.model,
action_spaces_dict=env.action_spaces_dict,
observation_spaces_dict=env.observation_spaces_dict,
agent_counts_dict=env.agent_counts_dict)
for pp in model_composer.policy.networks.values():
assert isinstance(pp, nn.Module)
if model_composer.critic:
for cc in model_composer.critic.networks.values():
assert isinstance(cc, nn.Module)
def test_initialize_config_module_ctx(hydra_restore_singletons: Any) -> None:
with initialize_config_module(
config_module="examples.jupyter_notebooks.cloud_app.conf"):
ret = compose(return_hydra_config=True)
assert ret.hydra.job.name == "app"
with initialize_config_module(
config_module="examples.jupyter_notebooks.cloud_app.conf",
job_name="test_job"):
ret = compose(return_hydra_config=True)
assert ret.hydra.job.name == "test_job"
with initialize_config_module(
config_module="examples.jupyter_notebooks.cloud_app.conf",
job_name="test_job"):
ret = compose(return_hydra_config=True)
assert ret.hydra.job.name == "test_job"
def test_with_initialize_config_module() -> None:
with initialize_config_module(version_base=None, config_module="hydra_app.conf"):
# config is relative to a module
cfg = compose(config_name="config", overrides=["app.user=test_user"])
assert cfg == {
"app": {"user": "******", "num1": 10, "num2": 20},
"db": {"host": "localhost", "port": 3306},
}
def test_initialize_config_module_ctx(
self, config_file: str, overrides: List[str], expected: Any
) -> None:
with initialize_config_module(
config_module="examples.jupyter_notebooks.cloud_app.conf",
job_name="job_name",
):
ret = compose(config_file, overrides)
assert ret == expected
def test_missing_init_py_error(hydra_restore_singletons: Any) -> None:
expected = (
"Primary config module 'hydra.test_utils.configs.missing_init_py' not found."
"\nCheck that it's correct and contains an __init__.py file")
with raises(Exception, match=re.escape(expected)):
with initialize_config_module(
config_module="hydra.test_utils.configs.missing_init_py"):
hydra = GlobalHydra.instance().hydra
assert hydra is not None
compose(config_name="test.yaml", overrides=[])
def construct_scene_optimizer(self) -> SceneOptimizer:
with hydra.initialize_config_module(config_module="gtsfm.configs"):
# config is relative to the gtsfm module
cfg = hydra.compose(
config_name=self.parsed_args.config_name,
overrides=[
"SceneOptimizer.multiview_optimizer.bundle_adjustment_module.shared_calib=True"
] if self.parsed_args.share_intrinsics else [],
)
logger.info("Using config: ")
logger.info(OmegaConf.to_yaml(cfg))
scene_optimizer: SceneOptimizer = instantiate(cfg.SceneOptimizer)
return scene_optimizer
def read_hydra_config(config_module: str,
config_name: str = None,
**hydra_overrides: str) -> DictConfig:
"""Read and assemble a hydra config, given the config module, name, and overrides.
:param config_module: Python module path of the hydra configuration package
:param config_name: Name of the defaults configuration yaml file within `config_module`
:param hydra_overrides: Overrides as kwargs, e.g. env="cartpole", configuration="test"
:return: Hydra DictConfig instance, assembled according to the given module, name, and overrides.
"""
with initialize_config_module(config_module):
cfg = compose(config_name, overrides=[key + "=" + value for key, value in hydra_overrides.items()])
return cfg
def test_create_computation_graph(self):
"""Will test Dask multi-processing capabilities and ability to serialize all objects."""
self.loader = OlssonLoader(str(DATA_ROOT_PATH / "set1_lund_door"),
image_extension="JPG")
with hydra.initialize_config_module(config_module="gtsfm.configs"):
# config is relative to the gtsfm module
cfg = hydra.compose(
config_name="scene_optimizer_unit_test_config.yaml")
scene_optimizer: SceneOptimizer = instantiate(cfg.SceneOptimizer)
# generate the dask computation graph
delayed_sfm_result, delayed_io = scene_optimizer.create_computation_graph(
num_images=len(self.loader),
image_pair_indices=self.loader.get_valid_pairs(),
image_graph=self.loader.create_computation_graph_for_images(),
all_intrinsics=self.loader.get_all_intrinsics(),
image_shapes=self.loader.get_image_shapes(),
absolute_pose_priors=self.loader.get_absolute_pose_priors(),
relative_pose_priors=self.loader.get_relative_pose_priors(
self.loader.get_valid_pairs()),
cameras_gt=self.loader.get_gt_cameras(),
gt_wTi_list=self.loader.get_gt_poses(),
)
# create dask client
cluster = LocalCluster(n_workers=1, threads_per_worker=4)
with Client(cluster):
sfm_result, *io = dask.compute(delayed_sfm_result, *delayed_io)
self.assertIsInstance(sfm_result, GtsfmData)
# compare the camera poses
computed_poses = sfm_result.get_camera_poses()
# get active cameras from largest connected component, may be <len(self.loader)
connected_camera_idxs = sfm_result.get_valid_camera_indices()
expected_poses = [
self.loader.get_camera_pose(i) for i in connected_camera_idxs
]
self.assertTrue(
comp_utils.compare_global_poses(computed_poses,
expected_poses,
trans_err_atol=1.0,
trans_err_rtol=0.1))
def compose_hydra_configuration(overrides: List[str]):
"""
Transform the list of overrides provided on the command line
to an actual VISSL configuration by merging these overrides
with the defaults configuration of VISSL
"""
assert_hydra_dependency()
# Backward compatibility with previous hydra versions:
# In Hydra 1.1 and above, the compose API is not experimental anymore
if get_hydra_version() >= (1, 1, 0):
from hydra import compose, initialize_config_module
else:
from hydra.experimental import compose, initialize_config_module
# Compose the overrides with "vissl/config/defaults.yaml"
with initialize_config_module(config_module="vissl.config"):
return compose("defaults", overrides=overrides)
def load_hydra_config(config_module: str, config_name: str,
hydra_overrides: Dict[str, str]) -> DictConfig:
"""Load a hydra config from a given config module + config name and additional hydra overrides.
:param config_module: The config module that should be used
:param config_name: The name of the config that should be used
:param hydra_overrides: The hydra overrides that should be applied
:return: A dict config of the created hydra config
"""
with initialize_config_module(config_module=config_module):
# Config is relative to a module
# For the HydraConfig init below, we need the hydra key there as well (=> return_hydra_config=True)
cfg = compose(config_name=config_name,
overrides=[
key + "=" + str(val)
for key, val in hydra_overrides.items()
])
return cfg
def test_sensor_dataset_config():
"""Ensure that config fields are populated correctly from YAML."""
dataset_name = "argoverse-v1.1"
with hydra.initialize_config_module(config_module="argoverse.config"):
cfg = hydra.compose(config_name=f"{dataset_name}.yaml")
argoverse_config: SensorDatasetConfig = instantiate(
cfg.SensorDatasetConfig)
# check a few camera names
assert argoverse_config.camera_sensors.has_camera("ring_rear_left")
assert argoverse_config.camera_sensors.has_camera("stereo_front_left_rect")
assert not argoverse_config.camera_sensors.has_camera(
"ring_rear_dummyname")
# check sample camera dimensions
assert argoverse_config.camera_sensors.ring_rear_left.img_width == 1920
assert argoverse_config.camera_sensors.ring_rear_left.img_height == 1200
# check other properties
assert argoverse_config.dataset_name == "argoverse-v1.1"
assert argoverse_config.ring_cam_fps == 30
assert argoverse_config.stereo_cam_fps == 5
def run_scene_optimizer(args: argparse.Namespace) -> None:
"""Run GTSFM over images from an Argoverse vehicle log"""
with hydra.initialize_config_module(config_module="gtsfm.configs"):
# config is relative to the gtsfm module
cfg = hydra.compose(config_name=args.config_name)
scene_optimizer: SceneOptimizer = instantiate(cfg.SceneOptimizer)
loader = ArgoverseDatasetLoader(
dataset_dir=args.dataset_dir,
log_id=args.log_id,
stride=args.stride,
max_num_imgs=args.max_num_imgs,
max_lookahead_sec=args.max_lookahead_sec,
camera_name=args.camera_name,
max_resolution=args.max_resolution,
)
delayed_sfm_result, delayed_io = scene_optimizer.create_computation_graph(
num_images=len(loader),
image_pair_indices=loader.get_valid_pairs(),
image_graph=loader.create_computation_graph_for_images(),
all_intrinsics=loader.get_all_intrinsics(),
image_shapes=loader.get_image_shapes(),
cameras_gt=loader.get_gt_cameras(),
)
# create dask client
cluster = LocalCluster(n_workers=args.num_workers,
threads_per_worker=args.threads_per_worker)
with Client(cluster), performance_report(filename="dask-report.html"):
sfm_result, *io = dask.compute(delayed_sfm_result, *delayed_io)
assert isinstance(sfm_result, GtsfmData)
scene_avg_reproj_error = sfm_result.get_avg_scene_reprojection_error()
logger.info("Scene avg reproj error: %.3f", scene_avg_reproj_error)
def get_all_configs_from_hydra(
default_conf: str,
all_hydra_config_modules: List[str]) -> List[pytest.param]:
"""Enumerate all environment configurations from the Hydra options.
:param default_conf: The name of the default config
:param all_hydra_config_modules: A list of hydra config modules
:return A list of pytest.param objects, to be used with @pytest.mark.parametrize
"""
configs = []
for config_module in all_hydra_config_modules:
# setup Hydra for the given config module
with initialize_config_module(config_module):
# query all argument overrides for this config module
for overrides in _get_all_overrides_from_hydra():
# add a single combination of module and hydra arguments to the list
config = pytest.param(config_module,
default_conf,
overrides,
id="-".join(overrides.values()))
configs.append(config)
return configs
def hydra_initialize_config_module() -> None:
initialize_config_module(version_base=None,
config_module="hydra.test_utils.configs")
def hydra_initialize_config_module() -> None:
initialize_config_module(config_module="hydra.test_utils.configs")
def test_user_logic(overrides: List[str], expected: int) -> None:
with initialize_config_module(version_base=None, config_module="hydra_app.conf"):
cfg = compose(config_name="config", overrides=overrides)
assert hydra_app.main.add(cfg.app, "num1", "num2") == expected
