def main() -> None: with initialize(config_path="conf"): cfg = compose(config_name="config", return_hydra_config=True) assert cfg.config == {"hello": "world"} assert cfg.hydra.job.name == "main" with initialize(config_path="conf", job_name="test_job"): cfg = compose(config_name="config", return_hydra_config=True) assert cfg.config == {"hello": "world"} assert cfg.hydra.job.name == "test_job" abs_config_dir = os.path.abspath("initialization_test_app/conf") with initialize_config_dir(config_dir=abs_config_dir): cfg = compose(config_name="config", return_hydra_config=True) assert cfg.config == {"hello": "world"} assert cfg.hydra.job.name == "app" with initialize_config_dir(config_dir=abs_config_dir, job_name="test_job"): cfg = compose(config_name="config", return_hydra_config=True) assert cfg.config == {"hello": "world"} assert cfg.hydra.job.name == "test_job" # Those tests can only work if the module is installed if len(sys.argv) > 1 and sys.argv[1] == "module_installed": with initialize_config_module(config_module="initialization_test_app.conf"): cfg = compose(config_name="config", return_hydra_config=True) assert cfg.config == {"hello": "world"} assert cfg.hydra.job.name == "app" with initialize_config_module( config_module="initialization_test_app.conf", job_name="test_job" ): cfg = compose(config_name="config", return_hydra_config=True) assert cfg.config == {"hello": "world"} assert cfg.hydra.job.name == "test_job"
def test_generated_config(self) -> None: with initialize_config_module(config_module="hydra_app.conf"): cfg = compose(config_name="config", overrides=["app.user=test_user"]) assert cfg == { "app": {"user": "******", "num1": 10, "num2": 20}, "db": {"host": "localhost", "port": 3306}, }
def test_initialize_with_module(hydra_restore_singletons: Any) -> None: with initialize_config_module( config_module="tests.test_apps.app_with_cfg_groups.conf", job_name="my_pp" ): assert compose(config_name="config") == { "optimizer": {"type": "nesterov", "lr": 0.001} }
def run_maze_job(hydra_overrides: Dict[str, str], config_module: str, config_name: str) -> DictConfig: """Runs rollout with the given config overrides using maze_run. :param hydra_overrides: Config overrides for hydra. :param config_module: The config module. :param config_name: The name of the default config. """ with initialize_config_module(config_module=config_module): # Config is relative to a module # For the HydraConfig init below, we need the hydra key there as well (=> return_hydra_config=True) cfg = compose(config_name=config_name, overrides=[key + "=" + str(val) for key, val in hydra_overrides.items()], return_hydra_config=True) # Init the HydraConfig: This is when Hydra actually creates the output dir and changes into it # (otherwise we only have the config object, but not the full run environment) HydraConfig.instance().set_config(cfg) # For the rollout itself, the Hydra config should not be there anymore with open_dict(cfg): del cfg["hydra"] # Run the rollout maze_run(cfg) return cfg
def check_env_and_model_instantiation(config_module: str, config: str, overrides: Dict[str, str]) -> None: """Check if env instantiation works.""" with initialize_config_module(config_module): # config is relative to a module cfg = compose( config, overrides=[key + "=" + value for key, value in overrides.items()]) env_factory = EnvFactory(cfg.env, cfg.wrappers if "wrappers" in cfg else {}) env = env_factory() assert env is not None assert isinstance(env, (StructuredEnv, StructuredEnvSpacesMixin)) if 'model' in overrides and overrides['model'] == 'rllib': return if 'model' in cfg: model_composer = Factory(BaseModelComposer).instantiate( cfg.model, action_spaces_dict=env.action_spaces_dict, observation_spaces_dict=env.observation_spaces_dict, agent_counts_dict=env.agent_counts_dict) for pp in model_composer.policy.networks.values(): assert isinstance(pp, nn.Module) if model_composer.critic: for cc in model_composer.critic.networks.values(): assert isinstance(cc, nn.Module)
def test_initialize_config_module_ctx(hydra_restore_singletons: Any) -> None: with initialize_config_module( config_module="examples.jupyter_notebooks.cloud_app.conf"): ret = compose(return_hydra_config=True) assert ret.hydra.job.name == "app" with initialize_config_module( config_module="examples.jupyter_notebooks.cloud_app.conf", job_name="test_job"): ret = compose(return_hydra_config=True) assert ret.hydra.job.name == "test_job" with initialize_config_module( config_module="examples.jupyter_notebooks.cloud_app.conf", job_name="test_job"): ret = compose(return_hydra_config=True) assert ret.hydra.job.name == "test_job"
def test_with_initialize_config_module() -> None: with initialize_config_module(version_base=None, config_module="hydra_app.conf"): # config is relative to a module cfg = compose(config_name="config", overrides=["app.user=test_user"]) assert cfg == { "app": {"user": "******", "num1": 10, "num2": 20}, "db": {"host": "localhost", "port": 3306}, }
def test_initialize_config_module_ctx( self, config_file: str, overrides: List[str], expected: Any ) -> None: with initialize_config_module( config_module="examples.jupyter_notebooks.cloud_app.conf", job_name="job_name", ): ret = compose(config_file, overrides) assert ret == expected
def test_missing_init_py_error(hydra_restore_singletons: Any) -> None: expected = ( "Primary config module 'hydra.test_utils.configs.missing_init_py' not found." "\nCheck that it's correct and contains an __init__.py file") with raises(Exception, match=re.escape(expected)): with initialize_config_module( config_module="hydra.test_utils.configs.missing_init_py"): hydra = GlobalHydra.instance().hydra assert hydra is not None compose(config_name="test.yaml", overrides=[])
def construct_scene_optimizer(self) -> SceneOptimizer: with hydra.initialize_config_module(config_module="gtsfm.configs"): # config is relative to the gtsfm module cfg = hydra.compose( config_name=self.parsed_args.config_name, overrides=[ "SceneOptimizer.multiview_optimizer.bundle_adjustment_module.shared_calib=True" ] if self.parsed_args.share_intrinsics else [], ) logger.info("Using config: ") logger.info(OmegaConf.to_yaml(cfg)) scene_optimizer: SceneOptimizer = instantiate(cfg.SceneOptimizer) return scene_optimizer
def read_hydra_config(config_module: str, config_name: str = None, **hydra_overrides: str) -> DictConfig: """Read and assemble a hydra config, given the config module, name, and overrides. :param config_module: Python module path of the hydra configuration package :param config_name: Name of the defaults configuration yaml file within `config_module` :param hydra_overrides: Overrides as kwargs, e.g. env="cartpole", configuration="test" :return: Hydra DictConfig instance, assembled according to the given module, name, and overrides. """ with initialize_config_module(config_module): cfg = compose(config_name, overrides=[key + "=" + value for key, value in hydra_overrides.items()]) return cfg
def test_create_computation_graph(self): """Will test Dask multi-processing capabilities and ability to serialize all objects.""" self.loader = OlssonLoader(str(DATA_ROOT_PATH / "set1_lund_door"), image_extension="JPG") with hydra.initialize_config_module(config_module="gtsfm.configs"): # config is relative to the gtsfm module cfg = hydra.compose( config_name="scene_optimizer_unit_test_config.yaml") scene_optimizer: SceneOptimizer = instantiate(cfg.SceneOptimizer) # generate the dask computation graph delayed_sfm_result, delayed_io = scene_optimizer.create_computation_graph( num_images=len(self.loader), image_pair_indices=self.loader.get_valid_pairs(), image_graph=self.loader.create_computation_graph_for_images(), all_intrinsics=self.loader.get_all_intrinsics(), image_shapes=self.loader.get_image_shapes(), absolute_pose_priors=self.loader.get_absolute_pose_priors(), relative_pose_priors=self.loader.get_relative_pose_priors( self.loader.get_valid_pairs()), cameras_gt=self.loader.get_gt_cameras(), gt_wTi_list=self.loader.get_gt_poses(), ) # create dask client cluster = LocalCluster(n_workers=1, threads_per_worker=4) with Client(cluster): sfm_result, *io = dask.compute(delayed_sfm_result, *delayed_io) self.assertIsInstance(sfm_result, GtsfmData) # compare the camera poses computed_poses = sfm_result.get_camera_poses() # get active cameras from largest connected component, may be <len(self.loader) connected_camera_idxs = sfm_result.get_valid_camera_indices() expected_poses = [ self.loader.get_camera_pose(i) for i in connected_camera_idxs ] self.assertTrue( comp_utils.compare_global_poses(computed_poses, expected_poses, trans_err_atol=1.0, trans_err_rtol=0.1))
def compose_hydra_configuration(overrides: List[str]): """ Transform the list of overrides provided on the command line to an actual VISSL configuration by merging these overrides with the defaults configuration of VISSL """ assert_hydra_dependency() # Backward compatibility with previous hydra versions: # In Hydra 1.1 and above, the compose API is not experimental anymore if get_hydra_version() >= (1, 1, 0): from hydra import compose, initialize_config_module else: from hydra.experimental import compose, initialize_config_module # Compose the overrides with "vissl/config/defaults.yaml" with initialize_config_module(config_module="vissl.config"): return compose("defaults", overrides=overrides)
def load_hydra_config(config_module: str, config_name: str, hydra_overrides: Dict[str, str]) -> DictConfig: """Load a hydra config from a given config module + config name and additional hydra overrides. :param config_module: The config module that should be used :param config_name: The name of the config that should be used :param hydra_overrides: The hydra overrides that should be applied :return: A dict config of the created hydra config """ with initialize_config_module(config_module=config_module): # Config is relative to a module # For the HydraConfig init below, we need the hydra key there as well (=> return_hydra_config=True) cfg = compose(config_name=config_name, overrides=[ key + "=" + str(val) for key, val in hydra_overrides.items() ]) return cfg
def test_sensor_dataset_config(): """Ensure that config fields are populated correctly from YAML.""" dataset_name = "argoverse-v1.1" with hydra.initialize_config_module(config_module="argoverse.config"): cfg = hydra.compose(config_name=f"{dataset_name}.yaml") argoverse_config: SensorDatasetConfig = instantiate( cfg.SensorDatasetConfig) # check a few camera names assert argoverse_config.camera_sensors.has_camera("ring_rear_left") assert argoverse_config.camera_sensors.has_camera("stereo_front_left_rect") assert not argoverse_config.camera_sensors.has_camera( "ring_rear_dummyname") # check sample camera dimensions assert argoverse_config.camera_sensors.ring_rear_left.img_width == 1920 assert argoverse_config.camera_sensors.ring_rear_left.img_height == 1200 # check other properties assert argoverse_config.dataset_name == "argoverse-v1.1" assert argoverse_config.ring_cam_fps == 30 assert argoverse_config.stereo_cam_fps == 5
def run_scene_optimizer(args: argparse.Namespace) -> None: """Run GTSFM over images from an Argoverse vehicle log""" with hydra.initialize_config_module(config_module="gtsfm.configs"): # config is relative to the gtsfm module cfg = hydra.compose(config_name=args.config_name) scene_optimizer: SceneOptimizer = instantiate(cfg.SceneOptimizer) loader = ArgoverseDatasetLoader( dataset_dir=args.dataset_dir, log_id=args.log_id, stride=args.stride, max_num_imgs=args.max_num_imgs, max_lookahead_sec=args.max_lookahead_sec, camera_name=args.camera_name, max_resolution=args.max_resolution, ) delayed_sfm_result, delayed_io = scene_optimizer.create_computation_graph( num_images=len(loader), image_pair_indices=loader.get_valid_pairs(), image_graph=loader.create_computation_graph_for_images(), all_intrinsics=loader.get_all_intrinsics(), image_shapes=loader.get_image_shapes(), cameras_gt=loader.get_gt_cameras(), ) # create dask client cluster = LocalCluster(n_workers=args.num_workers, threads_per_worker=args.threads_per_worker) with Client(cluster), performance_report(filename="dask-report.html"): sfm_result, *io = dask.compute(delayed_sfm_result, *delayed_io) assert isinstance(sfm_result, GtsfmData) scene_avg_reproj_error = sfm_result.get_avg_scene_reprojection_error() logger.info("Scene avg reproj error: %.3f", scene_avg_reproj_error)
def get_all_configs_from_hydra( default_conf: str, all_hydra_config_modules: List[str]) -> List[pytest.param]: """Enumerate all environment configurations from the Hydra options. :param default_conf: The name of the default config :param all_hydra_config_modules: A list of hydra config modules :return A list of pytest.param objects, to be used with @pytest.mark.parametrize """ configs = [] for config_module in all_hydra_config_modules: # setup Hydra for the given config module with initialize_config_module(config_module): # query all argument overrides for this config module for overrides in _get_all_overrides_from_hydra(): # add a single combination of module and hydra arguments to the list config = pytest.param(config_module, default_conf, overrides, id="-".join(overrides.values())) configs.append(config) return configs
def hydra_initialize_config_module() -> None: initialize_config_module(version_base=None, config_module="hydra.test_utils.configs")
def hydra_initialize_config_module() -> None: initialize_config_module(config_module="hydra.test_utils.configs")
def test_user_logic(overrides: List[str], expected: int) -> None: with initialize_config_module(version_base=None, config_module="hydra_app.conf"): cfg = compose(config_name="config", overrides=overrides) assert hydra_app.main.add(cfg.app, "num1", "num2") == expected