def test_user_logic(overrides: List[str], expected: int) -> None: try: initialize_with_module(calling_module="hydra_app.main", config_path="conf") cfg = compose(config_name="config", overrides=overrides) assert add(cfg.app, "num1", "num2") == expected finally: GlobalHydra.instance().clear()
def compose( config_name: Optional[str] = None, overrides: List[str] = [], strict: Optional[bool] = None, ) -> DictConfig: """ :param config_name: optional config name to load :param overrides: list of overrides for config file :param strict: optionally override the default strict mode :return: the composed config """ assert ( GlobalHydra().is_initialized() ), "GlobalHydra is not initialized, use @hydra.main() or call hydra.experimental.initialize() first" gh = GlobalHydra.instance() assert gh.hydra is not None cfg = gh.hydra.compose_config( config_name=config_name, overrides=overrides, strict=strict ) assert isinstance(cfg, DictConfig) if "hydra" in cfg: with open_dict(cfg): del cfg["hydra"] return cfg
def test_initialize() -> None: try: assert not GlobalHydra().is_initialized() initialize(config_dir=None, strict=True) assert GlobalHydra().is_initialized() finally: GlobalHydra().clear()
def compose( config_name: Optional[str] = None, overrides: List[str] = [], strict: Optional[bool] = None, return_hydra_config: bool = False, ) -> DictConfig: """ :param config_name: the name of the config (usually the file name without the .yaml extension) :param overrides: list of overrides for config file :param strict: optionally override the default strict mode :param return_hydra_config: True to return the hydra config node in the result :return: the composed config """ assert GlobalHydra().is_initialized(), ( "GlobalHydra is not initialized, use @hydra.main()" " or call one of the hydra.experimental initialize methods first") gh = GlobalHydra.instance() assert gh.hydra is not None cfg = gh.hydra.compose_config( config_name=config_name, overrides=overrides, run_mode=RunMode.RUN, strict=strict, from_shell=False, ) assert isinstance(cfg, DictConfig) if not return_hydra_config: if "hydra" in cfg: with open_dict(cfg): del cfg["hydra"] return cfg
def create_main_hydra2( cls, task_name: str, config_search_path: ConfigSearchPath, strict: Optional[bool], ) -> "Hydra": if strict is None: strict = True else: # DEPRECATED: remove in 1.1 msg = ( "\[email protected](strict) flag is deprecated and will removed in the next version." "\nSee https://hydra.cc/docs/next/upgrades/0.11_to_1.0/strict_mode_flag_deprecated" ) warnings.warn(message=msg, category=UserWarning) config_loader: ConfigLoader = ConfigLoaderImpl( config_search_path=config_search_path, default_strict=strict) hydra = cls(task_name=task_name, config_loader=config_loader) from hydra.core.global_hydra import GlobalHydra GlobalHydra.instance().initialize(hydra) return hydra
def hydra_instance() -> Union[Hydra, GlobalHydra]: "Provide Hydra/GlobalHydra instance for compose" if HydraConfig.initialized(): yield GlobalHydra.instance() hydra_init = initialize(config_path="../peddet/conf") yield hydra_init GlobalHydra.instance().clear()
def initialize_with_file_ctx(*args: Any, **kwargs: Any) -> Any: assert len(args) == 0, "Please use only named parameters" try: gh = copy.deepcopy(GlobalHydra.instance()) initialize_with_file(**kwargs) yield finally: GlobalHydra.set_instance(gh)
def test_strict_deprecation_warning(hydra_restore_singletons: Any) -> None: msg = ( "\[email protected](strict) flag is deprecated and will removed in the next version." "\nSee https://hydra.cc/docs/next/upgrades/0.11_to_1.0/strict_mode_flag_deprecated" ) with pytest.warns(expected_warning=UserWarning, match=re.escape(msg)): try: initialize(config_path=None, strict=True) finally: GlobalHydra.instance().clear()
def test_generated_config(self) -> None: try: # config is relative to a module initialize_with_module(calling_module="hydra_app.main", config_path="conf") cfg = compose(config_name="config", overrides=["app.user=test_user"]) assert cfg == { "app": {"user": "******", "num1": 10, "num2": 20}, "db": {"host": "localhost", "port": 3306}, } finally: GlobalHydra.instance().clear()
def main(cfg): OmegaConf.set_struct(cfg, False) # This allows getattr and hasattr methods to function correctly if cfg.pretty_print: print(OmegaConf.to_yaml(cfg)) trainer = Trainer(cfg) trainer.train() # # # https://github.com/facebookresearch/hydra/issues/440 GlobalHydra.get_state().clear() return 0
def test_initialize_with_config_path(hydra_restore_singletons: Any) -> None: assert not GlobalHydra().is_initialized() initialize(config_path="../hydra/test_utils/configs") assert GlobalHydra().is_initialized() gh = GlobalHydra.instance() assert gh.hydra is not None config_search_path = gh.hydra.config_loader.get_search_path() assert isinstance(config_search_path, ConfigSearchPathImpl) idx = config_search_path.find_first_match( SearchPathQuery(provider="main", path=None)) assert idx != -1
def initialize_ctx(*args: Any, **kwargs: Any) -> Any: assert len(args) == 0, "Please use only named parameters" try: gh = copy.deepcopy(GlobalHydra.instance()) caller_stack_depth = _default_caller_stack_depth if "caller_stack_depth" in kwargs: caller_stack_depth = kwargs["caller_stack_depth"] kwargs["caller_stack_depth"] = caller_stack_depth + 1 initialize(**kwargs) yield finally: GlobalHydra.set_instance(gh)
def create_main_hydra2( cls, task_name: str, config_search_path: ConfigSearchPath, ) -> "Hydra": config_loader: ConfigLoader = ConfigLoaderImpl( config_search_path=config_search_path) hydra = cls(task_name=task_name, config_loader=config_loader) from hydra.core.global_hydra import GlobalHydra GlobalHydra.instance().initialize(hydra) return hydra
def get_hydra_cfg(config_name: str = "default") -> omegaconf.DictConfig: """ Instantiate and return the hydra config -- streamlit and jupyter compatible Args: config_name: .yaml configuration name, without the extension Returns: The desired omegaconf.DictConfig """ GlobalHydra.instance().clear() hydra.experimental.initialize_config_dir(config_dir=str(PROJECT_ROOT / "conf")) return compose(config_name=config_name)
def test_initialize_with_config_dir() -> None: try: assert not GlobalHydra().is_initialized() initialize(config_dir="../hydra/test_utils/configs", strict=True) assert GlobalHydra().is_initialized() gh = GlobalHydra.instance() assert gh.hydra is not None config_search_path = gh.hydra.config_loader.get_search_path() assert isinstance(config_search_path, ConfigSearchPathImpl) idx = config_search_path.find_first_match( SearchPathQuery(provider="main", search_path=None)) assert idx != -1 finally: GlobalHydra().clear()
def test_initialize_old_version_base(hydra_restore_singletons: Any) -> None: assert not GlobalHydra().is_initialized() with raises( HydraException, match=f'version_base must be >= "{version.__compat_version__}"', ): initialize(version_base="1.0")
def test_config_installed( hydra_global_context: TGlobalHydraContext, # noqa: F811 ) -> None: with hydra_global_context(): config_loader = GlobalHydra.instance().config_loader() assert "my_default_output_dir" in config_loader.get_group_options( "hydra/output")
def test_initialize_bad_version_base(hydra_restore_singletons: Any) -> None: assert not GlobalHydra().is_initialized() with raises( TypeError, match="expected string or bytes-like object", ): initialize(version_base=1.1) # type: ignore
def convert_namespace_to_omegaconf(args: Namespace) -> DictConfig: # Here we are using field values provided in args to override counterparts inside config object overrides, deletes = override_module_args(args) cfg_name = "config" cfg_path = f"../../{cfg_name}" if not GlobalHydra().is_initialized(): initialize(config_path=cfg_path) composed_cfg = compose(cfg_name, overrides=overrides, strict=False) for k in deletes: composed_cfg[k] = None cfg = OmegaConf.create( OmegaConf.to_container(composed_cfg, resolve=True, enum_to_str=True)) # hack to be able to set Namespace in dict config. this should be removed when we update to newer # omegaconf version that supports object flags, or when we migrate all existing models from omegaconf import _utils old_primitive = _utils.is_primitive_type _utils.is_primitive_type = lambda _: True if cfg.task is None and getattr(args, "task", None): cfg.task = Namespace(**vars(args)) from fairseq.tasks import TASK_REGISTRY _set_legacy_defaults(cfg.task, TASK_REGISTRY[args.task]) cfg.task._name = args.task if cfg.model is None and getattr(args, "arch", None): cfg.model = Namespace(**vars(args)) from fairseq.models import ARCH_MODEL_REGISTRY _set_legacy_defaults(cfg.model, ARCH_MODEL_REGISTRY[args.arch]) cfg.model._name = args.arch if cfg.optimizer is None and getattr(args, "optimizer", None): cfg.optimizer = Namespace(**vars(args)) from fairseq.optim import OPTIMIZER_REGISTRY _set_legacy_defaults(cfg.optimizer, OPTIMIZER_REGISTRY[args.optimizer]) cfg.optimizer._name = args.optimizer if cfg.lr_scheduler is None and getattr(args, "lr_scheduler", None): cfg.lr_scheduler = Namespace(**vars(args)) from fairseq.optim.lr_scheduler import LR_SCHEDULER_REGISTRY _set_legacy_defaults(cfg.lr_scheduler, LR_SCHEDULER_REGISTRY[args.lr_scheduler]) cfg.lr_scheduler._name = args.lr_scheduler if cfg.criterion is None and getattr(args, "criterion", None): cfg.criterion = Namespace(**vars(args)) from fairseq.criterions import CRITERION_REGISTRY _set_legacy_defaults(cfg.criterion, CRITERION_REGISTRY[args.criterion]) cfg.criterion._name = args.criterion _utils.is_primitive_type = old_primitive OmegaConf.set_struct(cfg, True) return cfg
def __enter__(self) -> "SweepTaskFunction": overrides = copy.deepcopy(self.overrides) assert overrides is not None if self.temp_dir: Path(self.temp_dir).mkdir(parents=True, exist_ok=True) else: self.temp_dir = tempfile.mkdtemp() overrides.append(f"hydra.sweep.dir={self.temp_dir}") try: config_dir, config_name = split_config_path( self.config_path, self.config_name) job_name = detect_task_name(self.calling_file, self.calling_module) hydra_ = Hydra.create_main_hydra_file_or_module( calling_file=self.calling_file, calling_module=self.calling_module, config_path=config_dir, job_name=job_name, strict=self.strict, ) self.returns = hydra_.multirun( config_name=config_name, task_function=self, overrides=overrides, with_log_configuration=self.configure_logging, ) finally: GlobalHydra().clear() return self
def __enter__(self) -> "TaskTestFunction": try: config_dir, config_name = split_config_path( self.config_path, self.config_name) job_name = detect_task_name(self.calling_file, self.calling_module) self.hydra = Hydra.create_main_hydra_file_or_module( calling_file=self.calling_file, calling_module=self.calling_module, config_path=config_dir, job_name=job_name, strict=self.strict, ) self.temp_dir = tempfile.mkdtemp() overrides = copy.deepcopy(self.overrides) assert overrides is not None overrides.append(f"hydra.run.dir={self.temp_dir}") self.job_ret = self.hydra.run( config_name=config_name, task_function=self, overrides=overrides, with_log_configuration=self.configure_logging, ) return self finally: GlobalHydra().clear()
def compose( config_name: Optional[str] = None, overrides: List[str] = [], return_hydra_config: bool = False, strict: Optional[bool] = None, ) -> DictConfig: """ :param config_name: the name of the config (usually the file name without the .yaml extension) :param overrides: list of overrides for config file :param return_hydra_config: True to return the hydra config node in the result :param strict: DEPRECATED. If false, returned config has struct mode disabled. :return: the composed config """ assert ( GlobalHydra().is_initialized() ), "GlobalHydra is not initialized, use @hydra.main() or call one of the hydra initialization methods first" gh = GlobalHydra.instance() assert gh.hydra is not None cfg = gh.hydra.compose_config( config_name=config_name, overrides=overrides, run_mode=RunMode.RUN, from_shell=False, with_log_configuration=False, ) assert isinstance(cfg, DictConfig) if not return_hydra_config: if "hydra" in cfg: with open_dict(cfg): del cfg["hydra"] if strict is not None: # DEPRECATED: remove in 1.2 deprecation_warning( dedent( """ The strict flag in the compose API is deprecated and will be removed in the next version of Hydra. See https://hydra.cc/docs/upgrades/0.11_to_1.0/strict_mode_flag_deprecated for more info. """ ) ) OmegaConf.set_struct(cfg, strict) return cfg
def test_initialize_compat_version_base(hydra_restore_singletons: Any) -> None: assert not GlobalHydra().is_initialized() with raises( UserWarning, match=f"Will assume defaults for version {version.__compat_version__}", ): initialize() assert version.base_at_least(str(version.__compat_version__))
def test_config_installed() -> None: """ Tests that color options are available for both hydra/hydra_logging and hydra/job_logging """ with initialize(config_path="../hydra_plugins/hydra_colorlog/conf"): config_loader = GlobalHydra.instance().config_loader() assert "colorlog" in config_loader.get_group_options("hydra/job_logging") assert "colorlog" in config_loader.get_group_options("hydra/hydra_logging")
def test_config_in_dir() -> None: with initialize_ctx(config_path="../some_namespace/namespace_test/dir"): config_loader = GlobalHydra.instance().config_loader() assert "cifar10" in config_loader.get_group_options("dataset") assert "imagenet" in config_loader.get_group_options("dataset") assert "level1" in config_loader.list_groups("") assert "level2" in config_loader.list_groups("level1") assert "nested1" in config_loader.get_group_options("level1/level2") assert "nested2" in config_loader.get_group_options("level1/level2")
def test_config_in_dir(hydra_global_context: Any) -> None: with hydra_global_context(config_dir="../some_namespace/namespace_test/dir"): config_loader = GlobalHydra.instance().config_loader() assert "cifar10" in config_loader.get_group_options("dataset") assert "imagenet" in config_loader.get_group_options("dataset") assert "level1" in config_loader.list_groups("") assert "level2" in config_loader.list_groups("level1") assert "nested1" in config_loader.get_group_options("level1/level2") assert "nested2" in config_loader.get_group_options("level1/level2")
def test_missing_bad_config_dir_error(hydra_restore_singletons: Any) -> None: expected = ( "Primary config directory not found." "\nCheck that the config directory '/no_way_in_hell_1234567890' exists and readable" ) with pytest.raises(Exception, match=re.escape(expected)): with initialize_config_dir(config_dir="/no_way_in_hell_1234567890"): hydra = GlobalHydra.instance().hydra assert hydra is not None compose(config_name="test.yaml", overrides=[])
def test_missing_init_py_error(hydra_restore_singletons: Any) -> None: expected = ( "Primary config module 'hydra.test_utils.configs.missing_init_py' not found." "\nCheck that it's correct and contains an __init__.py file") with pytest.raises(Exception, match=re.escape(expected)): with initialize_config_module( config_module="hydra.test_utils.configs.missing_init_py"): hydra = GlobalHydra.instance().hydra assert hydra is not None compose(config_name="test.yaml", overrides=[])
def test_missing_init_py_error(hydra_restore_singletons: Any) -> None: with pytest.raises( Exception, match=re.escape( "Unexpected error checking content of 'hydra.test_utils.configs.missing_init_py', " "did you forget an __init__.py?"), ): with initialize_ctx( config_path="../hydra/test_utils/configs/missing_init_py"): hydra = GlobalHydra.instance().hydra assert hydra is not None hydra.compose_config(config_name=None, overrides=[])
def load_config(config_path:str='config', config_file:str='config', overrides:dict=None): """ Args: config_path (str): config_file (str): Return: config (dict): """ try: initialize(f'../../{config_path}') except ValueError: from hydra.core.global_hydra import GlobalHydra GlobalHydra.instance().clear() initialize(f'../../{config_path}') if overrides is not None: cfg = compose(config_name=config_file, overrides=overrides) else: cfg = compose(config_name=config_file) return cfg