def __init__(
self,
config_path: Optional[str] = _UNSPECIFIED_,
job_name: Optional[str] = None,
caller_stack_depth: int = 1,
) -> None:
self._gh_backup = get_gh_backup()
# DEPRECATED: remove in 1.2
# in 1.2, the default config_path should be changed to None
if config_path is _UNSPECIFIED_:
url = "https://hydra.cc/docs/next/upgrades/1.0_to_1.1/changes_to_hydra_main_config_path"
deprecation_warning(
message=dedent(f"""\
config_path is not specified in hydra.initialize().
See {url} for more information."""),
stacklevel=2,
)
config_path = "."
if config_path is not None and os.path.isabs(config_path):
raise HydraException(
"config_path in initialize() must be relative")
calling_file, calling_module = detect_calling_file_or_module_from_stack_frame(
caller_stack_depth + 1)
if job_name is None:
job_name = detect_task_name(calling_file=calling_file,
calling_module=calling_module)
Hydra.create_main_hydra_file_or_module(
calling_file=calling_file,
calling_module=calling_module,
config_path=config_path,
job_name=job_name,
)
def arguments_to_configuration(arguments: Dict) -> Dict:
"""Processes command line arguments into a dictionary.
Handles hydra file composition and parameter overwrites.
Arguments:
arguments: dictionary with all the cmd_line arguments passed to kiwi.
Returns:
Dictionary of the config imported from the config file.
"""
_reset_hydra()
config_file = Path(arguments['CONFIG_FILE'])
# Using Hydra
relative_dir = Path(
os.path.relpath(config_file.resolve().parent,
start=Path(__file__).parent))
Hydra.create_main_hydra_file_or_module(
calling_file=__file__,
calling_module=None,
config_dir=str(relative_dir),
strict=False,
)
config = hydra.experimental.compose(config_file=config_file.name,
overrides=arguments.get(
'OVERWRITES', []))
# Back to a dictionary
config_dict = OmegaConf.to_container(config)
config_dict['verbose'] = arguments.get('--verbose', False)
config_dict['quiet'] = arguments.get('--quiet', False)
return config_dict
def __init__(
self,
config_path: Optional[str] = None,
job_name: Optional[str] = None,
strict: Optional[bool] = None,
caller_stack_depth: int = 1,
) -> None:
self._gh_backup = get_gh_backup()
if config_path is not None and os.path.isabs(config_path):
raise HydraException(
"config_path in initialize() must be relative")
calling_file, calling_module = detect_calling_file_or_module_from_stack_frame(
caller_stack_depth + 1)
if job_name is None:
job_name = detect_task_name(calling_file=calling_file,
calling_module=calling_module)
Hydra.create_main_hydra_file_or_module(
calling_file=calling_file,
calling_module=calling_module,
config_path=config_path,
job_name=job_name,
strict=strict,
)
def initialize_with_file(file: Optional[str],
config_path: Optional[str] = None) -> None:
"""
Initialize Hydra and add the config_path to the search path.
The config path is relative to the calling_file.
:param file : The file to make the config_path relative to
:param config_path : The config path
"""
Hydra.create_main_hydra_file_or_module(file, None, config_path, None)
def __init__(self, config_module: str, job_name: str = "app"):
self._gh_backup = get_gh_backup()
Hydra.create_main_hydra_file_or_module(
calling_file=None,
calling_module=f"{config_module}.{job_name}",
config_path=None,
job_name=job_name,
)
def initialize(config_dir=None, strict=None, caller_stack_depth=1):
"""
:param config_dir: config directory relative to the calling script
:param strict:
:param caller_stack_depth:
:return:
"""
calling_file, calling_module = detect_calling_file_or_module(
caller_stack_depth + 1)
Hydra.create_main_hydra_file_or_module(calling_file, calling_module,
config_dir, strict)
def __init__(self, config_dir: str, job_name: str = "app") -> None:
self._gh_backup = get_gh_backup()
# Relative here would be interpreted as relative to cwd, which - depending on when it run
# may have unexpected meaning. best to force an absolute path to avoid confusion.
# Can consider using hydra.utils.to_absolute_path() to convert it at a future point if there is demand.
if not os.path.isabs(config_dir):
raise HydraException(
"initialize_config_dir() requires an absolute config_dir as input"
)
csp = create_config_search_path(search_path_dir=config_dir)
Hydra.create_main_hydra2(task_name=job_name, config_search_path=csp)
def initialize(
config_dir: Optional[str] = None,
strict: Optional[bool] = None,
caller_stack_depth: int = 1,
) -> None:
"""
:param config_dir: config directory relative to the calling script
:param strict:
:param caller_stack_depth:
:return:
"""
calling_file, calling_module = detect_calling_file_or_module_from_stack_frame(
caller_stack_depth + 1)
Hydra.create_main_hydra_file_or_module(calling_file, calling_module,
config_dir, strict)
def __init__(
self,
config_module: str,
version_base: Optional[str] = _UNSPECIFIED_,
job_name: str = "app",
):
self._gh_backup = get_gh_backup()
version.setbase(version_base)
Hydra.create_main_hydra_file_or_module(
calling_file=None,
calling_module=f"{config_module}.{job_name}",
config_path=None,
job_name=job_name,
)
def __enter__(self) -> "SweepTaskFunction":
overrides = copy.deepcopy(self.overrides)
assert overrides is not None
if self.temp_dir:
Path(self.temp_dir).mkdir(parents=True, exist_ok=True)
else:
self.temp_dir = tempfile.mkdtemp()
overrides.append(f"hydra.sweep.dir={self.temp_dir}")
try:
config_dir, config_name = split_config_path(
self.config_path, self.config_name)
job_name = detect_task_name(self.calling_file, self.calling_module)
hydra_ = Hydra.create_main_hydra_file_or_module(
calling_file=self.calling_file,
calling_module=self.calling_module,
config_path=config_dir,
job_name=job_name,
strict=self.strict,
)
self.returns = hydra_.multirun(
config_name=config_name,
task_function=self,
overrides=overrides,
with_log_configuration=self.configure_logging,
)
finally:
GlobalHydra().clear()
return self
def __enter__(self) -> "TaskTestFunction":
try:
config_dir, config_name = split_config_path(
self.config_path, self.config_name)
job_name = detect_task_name(self.calling_file, self.calling_module)
self.hydra = Hydra.create_main_hydra_file_or_module(
calling_file=self.calling_file,
calling_module=self.calling_module,
config_path=config_dir,
job_name=job_name,
strict=self.strict,
)
self.temp_dir = tempfile.mkdtemp()
overrides = copy.deepcopy(self.overrides)
assert overrides is not None
overrides.append(f"hydra.run.dir={self.temp_dir}")
self.job_ret = self.hydra.run(
config_name=config_name,
task_function=self,
overrides=overrides,
with_log_configuration=self.configure_logging,
)
return self
finally:
GlobalHydra().clear()
def initialize(
config_path: Optional[str] = None,
strict: Optional[bool] = None,
caller_stack_depth: int = _default_caller_stack_depth,
) -> None:
"""
Initialize automatically detect the calling file or module.
config_path is relative to the detected calling for or module.
:param config_path: A directory relative to the declaring python file or module
:param strict: (Deprecated), will be removed in the next major version
:param caller_stack_depth: stack depth of module the config_path is relative to
"""
calling_file, calling_module = detect_calling_file_or_module_from_stack_frame(
caller_stack_depth + 1)
Hydra.create_main_hydra_file_or_module(calling_file, calling_module,
config_path, strict)
def run_with_config(
task_function: Callable,
exp_handle: ExperimentDir,
config_path: pathlib.Path,
overrides: Sequence[str],
config_search_paths: Optional[Sequence[pathlib.Path]] = None,
) -> None:
setup_logging()
del config_search_paths # Hardcoded in hydra for now.
hydra_args = list(overrides) + [
f"hydra.run.dir={exp_handle.exp_path}",
f"job_id={exp_handle.exp_id}",
# Remove hydra defautl stuff.
"hydra.hydra_logging=null",
"hydra.job_logging=null",
"hydra.sweep=null",
"hydra.sweeper=null",
]
logging.info("Passing the following args to hydra: %s", hydra_args)
calling_file = "train.py"
abs_base_dir = os.path.realpath(os.path.dirname(calling_file))
hydra_config_path = str(config_path.resolve())
assert hydra_config_path.startswith(abs_base_dir + "/"), (
hydra_config_path,
abs_base_dir,
)
hydra_config_path = hydra_config_path[len(abs_base_dir) + 1 :]
hydra_obj = Hydra.create_main_hydra_file_or_module(
calling_file="run.py",
calling_module=None,
config_path=hydra_config_path,
task_function=task_function,
verbose="",
strict=False,
)
cfg = hydra_obj._load_config(copy.deepcopy(hydra_args))
use_slurm = False
if cfg.launcher is not None:
assert cfg.launcher.driver in ("slurm", "local"), cfg.launcher
use_slurm = cfg.launcher.driver == "slurm"
if use_slurm and not is_on_slurm():
logging.info("Config:\n%s", cfg.pretty())
executor = _build_slurm_executor(exp_handle, cfg.launcher)
job = executor.submit(hydra_obj.run, overrides=hydra_args)
exp_handle.save_job_id(job.job_id)
logging.info("Submitted job %s", job.job_id)
logging.info(
"stdout:\n\ttail -F %s/%s_0_log.out", exp_handle.slurm_path, job.job_id
)
logging.info(
"stderr:\n\ttail -F %s/%s_0_log.err", exp_handle.slurm_path, job.job_id
)
else:
exp_handle.save_job_id(LOCAL_JOB_ID)
hydra_obj.run(overrides=copy.deepcopy(hydra_args))
def load_from_yaml(self, config_path, strict=True):
config_dir, config_file = split_config_path(config_path)
strict = _strict_mode_strategy(strict, config_file)
search_path = create_automatic_config_search_path(
config_file, None, config_dir)
hydra = Hydra.create_main_hydra2(task_name='sdfs',
config_search_path=search_path,
strict=strict)
config = hydra.compose_config(config_file, [])
config.pop('hydra')
self.config = config
def get_hydra():
global_hydra = GlobalHydra()
if not global_hydra.is_initialized():
return Hydra.create_main_hydra_file_or_module(
calling_file=__file__,
calling_module=None,
config_dir="configs",
strict=False,
)
else:
return global_hydra.hydra
def _(calling_file, calling_module, config_path, overrides=None, strict=False):
task = TaskTestFunction()
hydra = Hydra(
calling_file=calling_file,
calling_module=calling_module,
config_path=config_path,
task_function=task,
verbose=None,
strict=strict,
)
task.hydra = hydra
task.overrides = overrides or []
return task
def _(calling_file, calling_module, config_path, overrides=None, strict=False):
sweep = SweepTaskFunction()
hydra = Hydra(
calling_file=calling_file,
calling_module=calling_module,
config_path=config_path,
task_function=sweep,
verbose=None,
strict=strict,
)
sweep.hydra = hydra
sweep.overrides = overrides or []
return sweep
def test_config_installed():
"""
Tests that color options are available for both hydra/hydra_logging and hydra/job_logging
"""
hydra = Hydra(
calling_file=None,
calling_module="hydra_plugins.hydra_colorlog",
config_path="conf/",
task_function=None,
strict=False,
)
assert "colorlog" in hydra.config_loader.get_group_options(
"hydra/job_logging")
assert "colorlog" in hydra.config_loader.get_group_options(
"hydra/hydra_logging")
def __enter__(self) -> "TaskTestFunction":
try:
config_dir, config_file = split_config_path(self.config_path)
self.hydra = Hydra.create_main_hydra_file_or_module(
calling_file=self.calling_file,
calling_module=self.calling_module,
config_dir=config_dir,
strict=self.strict,
)
self.temp_dir = tempfile.mkdtemp()
overrides = copy.deepcopy(self.overrides)
assert overrides is not None
overrides.append("hydra.run.dir={}".format(self.temp_dir))
self.job_ret = self.hydra.run(config_file=config_file,
task_function=self,
overrides=overrides)
return self
finally:
GlobalHydra().clear()
def __enter__(self):
try:
config_dir, config_file = split_config_path(self.config_path)
hydra = Hydra.create_main_hydra_file_or_module(
calling_file=self.calling_file,
calling_module=self.calling_module,
config_dir=config_dir,
strict=self.strict,
)
self.hydra = hydra
self.temp_dir = tempfile.mkdtemp()
overrides = copy.deepcopy(self.overrides)
overrides.append("hydra.run.dir={}".format(self.temp_dir))
self.job_ret = self.hydra.run(
config_file=config_file, task_function=self, overrides=overrides,
)
strip_node(self.job_ret.cfg, "hydra.run.dir")
return self
finally:
GlobalHydra().clear()
def __enter__(self) -> "SweepTaskFunction":
self.temp_dir = tempfile.mkdtemp()
overrides = copy.deepcopy(self.overrides)
assert overrides is not None
overrides.append(f"hydra.sweep.dir={self.temp_dir}")
try:
config_dir, config_name = split_config_path(
self.config_path, self.config_name)
hydra_ = Hydra.create_main_hydra_file_or_module(
calling_file=self.calling_file,
calling_module=self.calling_module,
config_dir=config_dir,
strict=self.strict,
)
self.returns = hydra_.multirun(config_name=config_name,
task_function=self,
overrides=overrides)
finally:
GlobalHydra().clear()
return self
def __enter__(self):
self.temp_dir = tempfile.mkdtemp()
overrides = copy.deepcopy(self.overrides)
overrides.append("hydra.sweep.dir={}".format(self.temp_dir))
try:
config_dir, config_file = split_config_path(self.config_path)
hydra = Hydra.create_main_hydra_file_or_module(
calling_file=self.calling_file,
calling_module=self.calling_module,
config_dir=config_dir,
strict=self.strict,
)
self.returns = hydra.multirun(
config_file=config_file, task_function=self, overrides=overrides
)
flat = [item for sublist in self.returns for item in sublist]
for ret in flat:
strip_node(ret.cfg, "hydra.sweep.dir")
finally:
GlobalHydra().clear()
return self