def test_user_logic(overrides: List[str], expected: int) -> None:
try:
initialize_with_module(calling_module="hydra_app.main", config_path="conf")
cfg = compose(config_name="config", overrides=overrides)
assert add(cfg.app, "num1", "num2") == expected
finally:
GlobalHydra.instance().clear()
def compose(
config_name: Optional[str] = None,
overrides: List[str] = [],
strict: Optional[bool] = None,
) -> DictConfig:
"""
:param config_name: optional config name to load
:param overrides: list of overrides for config file
:param strict: optionally override the default strict mode
:return: the composed config
"""
assert (
GlobalHydra().is_initialized()
), "GlobalHydra is not initialized, use @hydra.main() or call hydra.experimental.initialize() first"
gh = GlobalHydra.instance()
assert gh.hydra is not None
cfg = gh.hydra.compose_config(
config_name=config_name, overrides=overrides, strict=strict
)
assert isinstance(cfg, DictConfig)
if "hydra" in cfg:
with open_dict(cfg):
del cfg["hydra"]
return cfg
def test_initialize() -> None:
try:
assert not GlobalHydra().is_initialized()
initialize(config_dir=None, strict=True)
assert GlobalHydra().is_initialized()
finally:
GlobalHydra().clear()
def compose(
config_name: Optional[str] = None,
overrides: List[str] = [],
strict: Optional[bool] = None,
return_hydra_config: bool = False,
) -> DictConfig:
"""
:param config_name: the name of the config
(usually the file name without the .yaml extension)
:param overrides: list of overrides for config file
:param strict: optionally override the default strict mode
:param return_hydra_config: True to return the hydra config node in the result
:return: the composed config
"""
assert GlobalHydra().is_initialized(), (
"GlobalHydra is not initialized, use @hydra.main()"
" or call one of the hydra.experimental initialize methods first")
gh = GlobalHydra.instance()
assert gh.hydra is not None
cfg = gh.hydra.compose_config(
config_name=config_name,
overrides=overrides,
run_mode=RunMode.RUN,
strict=strict,
from_shell=False,
)
assert isinstance(cfg, DictConfig)
if not return_hydra_config:
if "hydra" in cfg:
with open_dict(cfg):
del cfg["hydra"]
return cfg
def create_main_hydra2(
cls,
task_name: str,
config_search_path: ConfigSearchPath,
strict: Optional[bool],
) -> "Hydra":
if strict is None:
strict = True
else:
# DEPRECATED: remove in 1.1
msg = (
"\[email protected](strict) flag is deprecated and will removed in the next version."
"\nSee https://hydra.cc/docs/next/upgrades/0.11_to_1.0/strict_mode_flag_deprecated"
)
warnings.warn(message=msg, category=UserWarning)
config_loader: ConfigLoader = ConfigLoaderImpl(
config_search_path=config_search_path, default_strict=strict)
hydra = cls(task_name=task_name, config_loader=config_loader)
from hydra.core.global_hydra import GlobalHydra
GlobalHydra.instance().initialize(hydra)
return hydra
def hydra_instance() -> Union[Hydra, GlobalHydra]:
"Provide Hydra/GlobalHydra instance for compose"
if HydraConfig.initialized():
yield GlobalHydra.instance()
hydra_init = initialize(config_path="../peddet/conf")
yield hydra_init
GlobalHydra.instance().clear()
def initialize_with_file_ctx(*args: Any, **kwargs: Any) -> Any:
assert len(args) == 0, "Please use only named parameters"
try:
gh = copy.deepcopy(GlobalHydra.instance())
initialize_with_file(**kwargs)
yield
finally:
GlobalHydra.set_instance(gh)
def test_strict_deprecation_warning(hydra_restore_singletons: Any) -> None:
msg = (
"\[email protected](strict) flag is deprecated and will removed in the next version."
"\nSee https://hydra.cc/docs/next/upgrades/0.11_to_1.0/strict_mode_flag_deprecated"
)
with pytest.warns(expected_warning=UserWarning, match=re.escape(msg)):
try:
initialize(config_path=None, strict=True)
finally:
GlobalHydra.instance().clear()
def test_generated_config(self) -> None:
try:
# config is relative to a module
initialize_with_module(calling_module="hydra_app.main", config_path="conf")
cfg = compose(config_name="config", overrides=["app.user=test_user"])
assert cfg == {
"app": {"user": "******", "num1": 10, "num2": 20},
"db": {"host": "localhost", "port": 3306},
}
finally:
GlobalHydra.instance().clear()
def main(cfg):
OmegaConf.set_struct(cfg, False) # This allows getattr and hasattr methods to function correctly
if cfg.pretty_print:
print(OmegaConf.to_yaml(cfg))
trainer = Trainer(cfg)
trainer.train()
#
# # https://github.com/facebookresearch/hydra/issues/440
GlobalHydra.get_state().clear()
return 0
def test_initialize_with_config_path(hydra_restore_singletons: Any) -> None:
assert not GlobalHydra().is_initialized()
initialize(config_path="../hydra/test_utils/configs")
assert GlobalHydra().is_initialized()
gh = GlobalHydra.instance()
assert gh.hydra is not None
config_search_path = gh.hydra.config_loader.get_search_path()
assert isinstance(config_search_path, ConfigSearchPathImpl)
idx = config_search_path.find_first_match(
SearchPathQuery(provider="main", path=None))
assert idx != -1
def initialize_ctx(*args: Any, **kwargs: Any) -> Any:
assert len(args) == 0, "Please use only named parameters"
try:
gh = copy.deepcopy(GlobalHydra.instance())
caller_stack_depth = _default_caller_stack_depth
if "caller_stack_depth" in kwargs:
caller_stack_depth = kwargs["caller_stack_depth"]
kwargs["caller_stack_depth"] = caller_stack_depth + 1
initialize(**kwargs)
yield
finally:
GlobalHydra.set_instance(gh)
def create_main_hydra2(
cls,
task_name: str,
config_search_path: ConfigSearchPath,
) -> "Hydra":
config_loader: ConfigLoader = ConfigLoaderImpl(
config_search_path=config_search_path)
hydra = cls(task_name=task_name, config_loader=config_loader)
from hydra.core.global_hydra import GlobalHydra
GlobalHydra.instance().initialize(hydra)
return hydra
def get_hydra_cfg(config_name: str = "default") -> omegaconf.DictConfig:
"""
Instantiate and return the hydra config -- streamlit and jupyter compatible
Args:
config_name: .yaml configuration name, without the extension
Returns:
The desired omegaconf.DictConfig
"""
GlobalHydra.instance().clear()
hydra.experimental.initialize_config_dir(config_dir=str(PROJECT_ROOT /
"conf"))
return compose(config_name=config_name)
def test_initialize_with_config_dir() -> None:
try:
assert not GlobalHydra().is_initialized()
initialize(config_dir="../hydra/test_utils/configs", strict=True)
assert GlobalHydra().is_initialized()
gh = GlobalHydra.instance()
assert gh.hydra is not None
config_search_path = gh.hydra.config_loader.get_search_path()
assert isinstance(config_search_path, ConfigSearchPathImpl)
idx = config_search_path.find_first_match(
SearchPathQuery(provider="main", search_path=None))
assert idx != -1
finally:
GlobalHydra().clear()
def test_initialize_old_version_base(hydra_restore_singletons: Any) -> None:
assert not GlobalHydra().is_initialized()
with raises(
HydraException,
match=f'version_base must be >= "{version.__compat_version__}"',
):
initialize(version_base="1.0")
def test_config_installed(
hydra_global_context: TGlobalHydraContext, # noqa: F811
) -> None:
with hydra_global_context():
config_loader = GlobalHydra.instance().config_loader()
assert "my_default_output_dir" in config_loader.get_group_options(
"hydra/output")
def test_initialize_bad_version_base(hydra_restore_singletons: Any) -> None:
assert not GlobalHydra().is_initialized()
with raises(
TypeError,
match="expected string or bytes-like object",
):
initialize(version_base=1.1) # type: ignore
def convert_namespace_to_omegaconf(args: Namespace) -> DictConfig:
# Here we are using field values provided in args to override counterparts inside config object
overrides, deletes = override_module_args(args)
cfg_name = "config"
cfg_path = f"../../{cfg_name}"
if not GlobalHydra().is_initialized():
initialize(config_path=cfg_path)
composed_cfg = compose(cfg_name, overrides=overrides, strict=False)
for k in deletes:
composed_cfg[k] = None
cfg = OmegaConf.create(
OmegaConf.to_container(composed_cfg, resolve=True, enum_to_str=True))
# hack to be able to set Namespace in dict config. this should be removed when we update to newer
# omegaconf version that supports object flags, or when we migrate all existing models
from omegaconf import _utils
old_primitive = _utils.is_primitive_type
_utils.is_primitive_type = lambda _: True
if cfg.task is None and getattr(args, "task", None):
cfg.task = Namespace(**vars(args))
from fairseq.tasks import TASK_REGISTRY
_set_legacy_defaults(cfg.task, TASK_REGISTRY[args.task])
cfg.task._name = args.task
if cfg.model is None and getattr(args, "arch", None):
cfg.model = Namespace(**vars(args))
from fairseq.models import ARCH_MODEL_REGISTRY
_set_legacy_defaults(cfg.model, ARCH_MODEL_REGISTRY[args.arch])
cfg.model._name = args.arch
if cfg.optimizer is None and getattr(args, "optimizer", None):
cfg.optimizer = Namespace(**vars(args))
from fairseq.optim import OPTIMIZER_REGISTRY
_set_legacy_defaults(cfg.optimizer, OPTIMIZER_REGISTRY[args.optimizer])
cfg.optimizer._name = args.optimizer
if cfg.lr_scheduler is None and getattr(args, "lr_scheduler", None):
cfg.lr_scheduler = Namespace(**vars(args))
from fairseq.optim.lr_scheduler import LR_SCHEDULER_REGISTRY
_set_legacy_defaults(cfg.lr_scheduler,
LR_SCHEDULER_REGISTRY[args.lr_scheduler])
cfg.lr_scheduler._name = args.lr_scheduler
if cfg.criterion is None and getattr(args, "criterion", None):
cfg.criterion = Namespace(**vars(args))
from fairseq.criterions import CRITERION_REGISTRY
_set_legacy_defaults(cfg.criterion, CRITERION_REGISTRY[args.criterion])
cfg.criterion._name = args.criterion
_utils.is_primitive_type = old_primitive
OmegaConf.set_struct(cfg, True)
return cfg
def __enter__(self) -> "SweepTaskFunction":
overrides = copy.deepcopy(self.overrides)
assert overrides is not None
if self.temp_dir:
Path(self.temp_dir).mkdir(parents=True, exist_ok=True)
else:
self.temp_dir = tempfile.mkdtemp()
overrides.append(f"hydra.sweep.dir={self.temp_dir}")
try:
config_dir, config_name = split_config_path(
self.config_path, self.config_name)
job_name = detect_task_name(self.calling_file, self.calling_module)
hydra_ = Hydra.create_main_hydra_file_or_module(
calling_file=self.calling_file,
calling_module=self.calling_module,
config_path=config_dir,
job_name=job_name,
strict=self.strict,
)
self.returns = hydra_.multirun(
config_name=config_name,
task_function=self,
overrides=overrides,
with_log_configuration=self.configure_logging,
)
finally:
GlobalHydra().clear()
return self
def __enter__(self) -> "TaskTestFunction":
try:
config_dir, config_name = split_config_path(
self.config_path, self.config_name)
job_name = detect_task_name(self.calling_file, self.calling_module)
self.hydra = Hydra.create_main_hydra_file_or_module(
calling_file=self.calling_file,
calling_module=self.calling_module,
config_path=config_dir,
job_name=job_name,
strict=self.strict,
)
self.temp_dir = tempfile.mkdtemp()
overrides = copy.deepcopy(self.overrides)
assert overrides is not None
overrides.append(f"hydra.run.dir={self.temp_dir}")
self.job_ret = self.hydra.run(
config_name=config_name,
task_function=self,
overrides=overrides,
with_log_configuration=self.configure_logging,
)
return self
finally:
GlobalHydra().clear()
def compose(
config_name: Optional[str] = None,
overrides: List[str] = [],
return_hydra_config: bool = False,
strict: Optional[bool] = None,
) -> DictConfig:
"""
:param config_name: the name of the config
(usually the file name without the .yaml extension)
:param overrides: list of overrides for config file
:param return_hydra_config: True to return the hydra config node in the result
:param strict: DEPRECATED. If false, returned config has struct mode disabled.
:return: the composed config
"""
assert (
GlobalHydra().is_initialized()
), "GlobalHydra is not initialized, use @hydra.main() or call one of the hydra initialization methods first"
gh = GlobalHydra.instance()
assert gh.hydra is not None
cfg = gh.hydra.compose_config(
config_name=config_name,
overrides=overrides,
run_mode=RunMode.RUN,
from_shell=False,
with_log_configuration=False,
)
assert isinstance(cfg, DictConfig)
if not return_hydra_config:
if "hydra" in cfg:
with open_dict(cfg):
del cfg["hydra"]
if strict is not None:
# DEPRECATED: remove in 1.2
deprecation_warning(
dedent(
"""
The strict flag in the compose API is deprecated and will be removed in the next version of Hydra.
See https://hydra.cc/docs/upgrades/0.11_to_1.0/strict_mode_flag_deprecated for more info.
"""
)
)
OmegaConf.set_struct(cfg, strict)
return cfg
def test_initialize_compat_version_base(hydra_restore_singletons: Any) -> None:
assert not GlobalHydra().is_initialized()
with raises(
UserWarning,
match=f"Will assume defaults for version {version.__compat_version__}",
):
initialize()
assert version.base_at_least(str(version.__compat_version__))
def test_config_installed() -> None:
"""
Tests that color options are available for both hydra/hydra_logging and hydra/job_logging
"""
with initialize(config_path="../hydra_plugins/hydra_colorlog/conf"):
config_loader = GlobalHydra.instance().config_loader()
assert "colorlog" in config_loader.get_group_options("hydra/job_logging")
assert "colorlog" in config_loader.get_group_options("hydra/hydra_logging")
def test_config_in_dir() -> None:
with initialize_ctx(config_path="../some_namespace/namespace_test/dir"):
config_loader = GlobalHydra.instance().config_loader()
assert "cifar10" in config_loader.get_group_options("dataset")
assert "imagenet" in config_loader.get_group_options("dataset")
assert "level1" in config_loader.list_groups("")
assert "level2" in config_loader.list_groups("level1")
assert "nested1" in config_loader.get_group_options("level1/level2")
assert "nested2" in config_loader.get_group_options("level1/level2")
def test_config_in_dir(hydra_global_context: Any) -> None:
with hydra_global_context(config_dir="../some_namespace/namespace_test/dir"):
config_loader = GlobalHydra.instance().config_loader()
assert "cifar10" in config_loader.get_group_options("dataset")
assert "imagenet" in config_loader.get_group_options("dataset")
assert "level1" in config_loader.list_groups("")
assert "level2" in config_loader.list_groups("level1")
assert "nested1" in config_loader.get_group_options("level1/level2")
assert "nested2" in config_loader.get_group_options("level1/level2")
def test_missing_bad_config_dir_error(hydra_restore_singletons: Any) -> None:
expected = (
"Primary config directory not found."
"\nCheck that the config directory '/no_way_in_hell_1234567890' exists and readable"
)
with pytest.raises(Exception, match=re.escape(expected)):
with initialize_config_dir(config_dir="/no_way_in_hell_1234567890"):
hydra = GlobalHydra.instance().hydra
assert hydra is not None
compose(config_name="test.yaml", overrides=[])
def test_missing_init_py_error(hydra_restore_singletons: Any) -> None:
expected = (
"Primary config module 'hydra.test_utils.configs.missing_init_py' not found."
"\nCheck that it's correct and contains an __init__.py file")
with pytest.raises(Exception, match=re.escape(expected)):
with initialize_config_module(
config_module="hydra.test_utils.configs.missing_init_py"):
hydra = GlobalHydra.instance().hydra
assert hydra is not None
compose(config_name="test.yaml", overrides=[])
def test_missing_init_py_error(hydra_restore_singletons: Any) -> None:
with pytest.raises(
Exception,
match=re.escape(
"Unexpected error checking content of 'hydra.test_utils.configs.missing_init_py', "
"did you forget an __init__.py?"),
):
with initialize_ctx(
config_path="../hydra/test_utils/configs/missing_init_py"):
hydra = GlobalHydra.instance().hydra
assert hydra is not None
hydra.compose_config(config_name=None, overrides=[])
def load_config(config_path:str='config', config_file:str='config', overrides:dict=None):
"""
Args:
config_path (str):
config_file (str):
Return:
config (dict):
"""
try:
initialize(f'../../{config_path}')
except ValueError:
from hydra.core.global_hydra import GlobalHydra
GlobalHydra.instance().clear()
initialize(f'../../{config_path}')
if overrides is not None:
cfg = compose(config_name=config_file, overrides=overrides)
else:
cfg = compose(config_name=config_file)
return cfg
