def _merge_config(self, cfg: DictConfig, family: str, name: str, required: bool) -> DictConfig: try: if family != "": new_cfg = f"{family}/{name}" else: new_cfg = name loaded_cfg, _ = self._load_config_impl(new_cfg) if loaded_cfg is None: if required: if family == "": msg = f"Could not load {new_cfg}" raise MissingConfigException(msg, new_cfg) else: options = self.get_group_options(family) if options: lst = "\n\t".join(options) msg = f"Could not load {new_cfg}, available options:\n{family}:\n\t{lst}" else: msg = f"Could not load {new_cfg}" raise MissingConfigException(msg, new_cfg, options) else: return cfg else: ret = OmegaConf.merge(cfg, loaded_cfg) assert isinstance(ret, DictConfig) return ret except OmegaConfBaseException as ex: raise HydraException(f"Error merging {family}={name}") from ex
def _merge_config( self, cfg: DictConfig, family: str, name: str, required: bool ) -> DictConfig: if family != "": new_cfg = "{}/{}".format(family, name) else: new_cfg = name loaded_cfg = self._load_config_impl(new_cfg) if loaded_cfg is None: if required: if family == "": msg = "Could not load {}".format(new_cfg) raise MissingConfigException(msg, new_cfg) else: options = self.get_group_options(family) if options: msg = "Could not load {}, available options:\n{}:\n\t{}".format( new_cfg, family, "\n\t".join(options) ) else: msg = "Could not load {}".format(new_cfg) raise MissingConfigException(msg, new_cfg, options) else: return cfg else: ret = OmegaConf.merge(cfg, loaded_cfg) assert isinstance(ret, DictConfig) return ret
def missing_config_error(repo: IConfigRepository, element: DefaultElement) -> None: options = None if element.config_group is not None: options = repo.get_group_options(element.config_group, ObjectType.CONFIG) opt_list = "\n".join(["\t" + x for x in options]) msg = ( f"Could not find '{element.config_name}' in the config group '{element.config_group}'" f"\nAvailable options:\n{opt_list}\n" ) else: msg = dedent( f"""\ Could not load {element.config_path()}. """ ) descs = [] for src in repo.get_sources(): descs.append(f"\t{repr(src)}") lines = "\n".join(descs) msg += "\nConfig search path:" + f"\n{lines}" raise MissingConfigException( missing_cfg_file=element.config_path(), message=msg, options=options, )
def _merge_config( self, cfg: DictConfig, config_group: str, name: str, required: bool, is_primary_config: bool, package_override: Optional[str], ) -> DictConfig: try: if config_group != "": new_cfg = f"{config_group}/{name}" else: new_cfg = name loaded_cfg, _ = self._load_config_impl( new_cfg, is_primary_config=is_primary_config, package_override=package_override, ) if loaded_cfg is None: if required: if config_group == "": msg = f"Could not load {new_cfg}" raise MissingConfigException(msg, new_cfg) else: options = self.get_group_options(config_group) if options: opt_list = "\n".join(["\t" + x for x in options]) msg = ( f"Could not load {new_cfg}.\nAvailable options:" f"\n{opt_list}" ) else: msg = f"Could not load {new_cfg}" raise MissingConfigException(msg, new_cfg, options) else: return cfg else: ret = OmegaConf.merge(cfg, loaded_cfg) assert isinstance(ret, DictConfig) return ret except OmegaConfBaseException as ex: raise ConfigCompositionException( f"Error merging {config_group}={name}" ) from ex
def _missing_config_error(self, config_name: Optional[str], msg: str, with_search_path: bool) -> None: def add_search_path() -> str: descs = [] for src in self.repository.get_sources(): if src.provider != "schema": descs.append(f"\t{repr(src)}") lines = "\n".join(descs) if with_search_path: return msg + "\nSearch path:" + f"\n{lines}" else: return msg raise MissingConfigException(missing_cfg_file=config_name, message=add_search_path())
def __init__(self, task_name: str, config_loader: ConfigLoader) -> None: """ :param task_name: task name :param config_loader: config loader """ setup_globals() self.config_loader = config_loader for source in config_loader.get_sources(): # if specified, make sure main config search path exists if source.provider == "main": if not source.exists(""): raise MissingConfigException( missing_cfg_file=source.path, message=f"Primary config dir not found: {source}", ) JobRuntime().set("name", task_name)
def compose_config( self, config_name: Optional[str], overrides: List[str], strict: Optional[bool] = None, with_log_configuration: bool = False, ) -> DictConfig: """ :param self: :param config_name: :param overrides: :param with_log_configuration: True to configure logging subsystem from the loaded config :param strict: None for default behavior (default to true for config file, false if no config file). otherwise forces specific behavior. :return: """ for source in self.config_loader.get_sources(): # if specified, make sure main config search path exists if source.provider == "main": if not source.exists(""): raise MissingConfigException( missing_cfg_file=source.path, message=f"Primary config dir not found: {source}", ) cfg = self.config_loader.load_configuration(config_name=config_name, overrides=overrides, strict=strict) with open_dict(cfg): from hydra import __version__ cfg.hydra.runtime.version = __version__ cfg.hydra.runtime.cwd = os.getcwd() if with_log_configuration: configure_log(cfg.hydra.hydra_logging, cfg.hydra.verbose) global log log = logging.getLogger(__name__) self._print_debug_info() return cfg
def load_configuration( self, config_file: Optional[str], overrides: List[str], strict: Optional[bool] = None, ) -> DictConfig: assert config_file is None or isinstance(config_file, str) assert strict is None or isinstance(strict, bool) assert isinstance(overrides, list) if strict is None: strict = self.default_strict assert overrides is None or isinstance(overrides, list) overrides = copy.deepcopy(overrides) or [] if config_file is not None and not self.exists_in_search_path(config_file): raise MissingConfigException( missing_cfg_file=config_file, message="Cannot find primary config file: {}\nSearch path:\n{}".format( config_file, "\n".join( [ "\t{} (from {})".format(src.path, src.provider) for src in self.repository.sources ] ), ), ) # Load hydra config hydra_cfg = self._create_cfg(cfg_filename="hydra.yaml") # Load job config job_cfg = self._create_cfg(cfg_filename=config_file, record_load=False) defaults = ConfigLoader._get_defaults(hydra_cfg) if config_file is not None: defaults.append(config_file) split_at = len(defaults) job_defaults = ConfigLoader._get_defaults(job_cfg) ConfigLoader._merge_default_lists(defaults, job_defaults) consumed = self._apply_defaults_overrides(overrides, defaults) consumed_free_job_defaults = self._apply_free_defaults(defaults, overrides) ConfigLoader._validate_defaults(defaults) # Load and defaults and merge them into cfg cfg = self._merge_defaults(hydra_cfg, defaults, split_at) OmegaConf.set_struct(cfg.hydra, True) OmegaConf.set_struct(cfg, strict) # Merge all command line overrides after enabling strict flag all_consumed = consumed + consumed_free_job_defaults remaining_overrides = [x for x in overrides if x not in all_consumed] cfg.merge_with_dotlist(remaining_overrides) remaining = consumed + consumed_free_job_defaults + remaining_overrides def is_hydra(x: str) -> bool: return x.startswith("hydra.") or x.startswith("hydra/") cfg.hydra.overrides.task = [x for x in remaining if not is_hydra(x)] cfg.hydra.overrides.hydra = [x for x in remaining if is_hydra(x)] with open_dict(cfg.hydra.job): if "name" not in cfg.hydra.job: cfg.hydra.job.name = JobRuntime().get("name") cfg.hydra.job.override_dirname = get_overrides_dirname( input_list=cfg.hydra.overrides.task, kv_sep=cfg.hydra.job.config.override_dirname.kv_sep, item_sep=cfg.hydra.job.config.override_dirname.item_sep, exclude_keys=cfg.hydra.job.config.override_dirname.exclude_keys, ) cfg.hydra.job.config_file = config_file return cfg
def load_configuration( self, config_name: Optional[str], overrides: List[str], strict: Optional[bool] = None, ) -> DictConfig: if strict is None: strict = self.default_strict parsed_overrides = [ self._parse_override(override) for override in overrides ] if config_name is not None and not self.repository.config_exists( config_name): # TODO: handle schema as a special case descs = [ f"\t{src.path} (from {src.provider})" for src in self.repository.get_sources() ] lines = "\n".join(descs) raise MissingConfigException( missing_cfg_file=config_name, message= f"Cannot find primary config file: {config_name}\nSearch path:\n{lines}", ) # Load hydra config hydra_cfg, _load_trace = self._load_primary_config( cfg_filename="hydra_config") # Load job config job_cfg, job_cfg_load_trace = self._load_primary_config( cfg_filename=config_name, record_load=False) job_defaults = self._parse_defaults(job_cfg) defaults = self._parse_defaults(hydra_cfg) job_cfg_type = OmegaConf.get_type(job_cfg) if job_cfg_type is not None and not issubclass(job_cfg_type, dict): hydra_cfg._promote(job_cfg_type) # this is breaking encapsulation a bit. can potentially be implemented in OmegaConf hydra_cfg._metadata.ref_type = job_cfg._metadata.ref_type # if defaults are re-introduced by the promotion, remove it. if "defaults" in hydra_cfg: with open_dict(hydra_cfg): del hydra_cfg["defaults"] if config_name is not None: defaults.append( DefaultElement(config_group=None, config_name="__SELF__")) split_at = len(defaults) config_group_overrides, config_overrides = self.split_overrides( parsed_overrides) self._combine_default_lists(defaults, job_defaults) ConfigLoaderImpl._apply_overrides_to_defaults(config_group_overrides, defaults) # Load and defaults and merge them into cfg cfg = self._merge_defaults_into_config(hydra_cfg, job_cfg, job_cfg_load_trace, defaults, split_at) OmegaConf.set_struct(cfg.hydra, True) OmegaConf.set_struct(cfg, strict) # Merge all command line overrides after enabling strict flag ConfigLoaderImpl._apply_overrides_to_config( [x.input_line for x in config_overrides], cfg) app_overrides = [] for pwl in parsed_overrides: override = pwl.override assert override.key is not None key = override.key if key.startswith("hydra.") or key.startswith("hydra/"): cfg.hydra.overrides.hydra.append(pwl.input_line) else: cfg.hydra.overrides.task.append(pwl.input_line) app_overrides.append(pwl) with open_dict(cfg.hydra.job): if "name" not in cfg.hydra.job: cfg.hydra.job.name = JobRuntime().get("name") cfg.hydra.job.override_dirname = get_overrides_dirname( input_list=app_overrides, kv_sep=cfg.hydra.job.config.override_dirname.kv_sep, item_sep=cfg.hydra.job.config.override_dirname.item_sep, exclude_keys=cfg.hydra.job.config.override_dirname. exclude_keys, ) cfg.hydra.job.config_name = config_name for key in cfg.hydra.job.env_copy: cfg.hydra.job.env_set[key] = os.environ[key] return cfg
def load_configuration( self, config_name: Optional[str], overrides: List[str], strict: Optional[bool] = None, ) -> DictConfig: assert config_name is None or isinstance(config_name, str) assert strict is None or isinstance(strict, bool) assert isinstance(overrides, list) if strict is None: strict = self.default_strict assert overrides is None or isinstance(overrides, list) overrides = copy.deepcopy(overrides) or [] if config_name is not None and not self.exists_in_search_path( config_name): # TODO: handle schema as a special case descs = [ "\t{} (from {})".format(src.path, src.provider) for src in self.repository.get_sources() ] lines = "\n".join(descs) raise MissingConfigException( missing_cfg_file=config_name, message= f"Cannot find primary config file: {config_name}\nSearch path:\n{lines}", ) # Load hydra config hydra_cfg, _load_trace = self._create_cfg(cfg_filename="hydra_config") # Load job config job_cfg, job_cfg_load_trace = self._create_cfg( cfg_filename=config_name, record_load=False) job_defaults = ConfigLoaderImpl._get_defaults(job_cfg) defaults = ConfigLoaderImpl._get_defaults(hydra_cfg) job_cfg_type = OmegaConf.get_type(job_cfg) if job_cfg_type is not None and not issubclass(job_cfg_type, dict): hydra_cfg._promote(job_cfg_type) # this is breaking encapsulation a bit. can potentially be implemented in OmegaConf hydra_cfg._metadata.ref_type = job_cfg._metadata.ref_type # if defaults are re-introduced by the promotion, remove it. if "defaults" in hydra_cfg: with open_dict(hydra_cfg): del hydra_cfg["defaults"] if config_name is not None: defaults.append("__SELF__") split_at = len(defaults) ConfigLoaderImpl._merge_default_lists(defaults, job_defaults) consumed = self._apply_defaults_overrides(overrides, defaults) consumed_free_job_defaults = self._apply_free_defaults( defaults, overrides) ConfigLoaderImpl._validate_defaults(defaults) # Load and defaults and merge them into cfg cfg = self._merge_defaults(hydra_cfg, job_cfg, job_cfg_load_trace, defaults, split_at) OmegaConf.set_struct(cfg.hydra, True) OmegaConf.set_struct(cfg, strict) # Merge all command line overrides after enabling strict flag all_consumed = consumed + consumed_free_job_defaults remaining_overrides = [x for x in overrides if x not in all_consumed] cfg.merge_with_dotlist(remaining_overrides) remaining = consumed + consumed_free_job_defaults + remaining_overrides def is_hydra(x: str) -> bool: return x.startswith("hydra.") or x.startswith("hydra/") cfg.hydra.overrides.task = [x for x in remaining if not is_hydra(x)] cfg.hydra.overrides.hydra = [x for x in remaining if is_hydra(x)] with open_dict(cfg.hydra.job): if "name" not in cfg.hydra.job: cfg.hydra.job.name = JobRuntime().get("name") cfg.hydra.job.override_dirname = get_overrides_dirname( input_list=cfg.hydra.overrides.task, kv_sep=cfg.hydra.job.config.override_dirname.kv_sep, item_sep=cfg.hydra.job.config.override_dirname.item_sep, exclude_keys=cfg.hydra.job.config.override_dirname. exclude_keys, ) cfg.hydra.job.config_name = config_name return cfg
def load_configuration( self, config_name: Optional[str], overrides: List[str], strict: Optional[bool] = None, ) -> DictConfig: if strict is None: strict = self.default_strict overrides = copy.deepcopy(overrides) or [] if config_name is not None and not self.exists_in_search_path( config_name): # TODO: handle schema as a special case descs = [ f"\t{src.path} (from {src.provider})" for src in self.repository.get_sources() ] lines = "\n".join(descs) raise MissingConfigException( missing_cfg_file=config_name, message= f"Cannot find primary config file: {config_name}\nSearch path:\n{lines}", ) # Load hydra config hydra_cfg, _load_trace = self._create_cfg(cfg_filename="hydra_config") # Load job config job_cfg, job_cfg_load_trace = self._create_cfg( cfg_filename=config_name, record_load=False) job_defaults = self._parse_defaults(job_cfg) defaults = self._parse_defaults(hydra_cfg) job_cfg_type = OmegaConf.get_type(job_cfg) if job_cfg_type is not None and not issubclass(job_cfg_type, dict): hydra_cfg._promote(job_cfg_type) # this is breaking encapsulation a bit. can potentially be implemented in OmegaConf hydra_cfg._metadata.ref_type = job_cfg._metadata.ref_type # if defaults are re-introduced by the promotion, remove it. if "defaults" in hydra_cfg: with open_dict(hydra_cfg): del hydra_cfg["defaults"] if config_name is not None: defaults.append( DefaultElement(config_group=None, config_name="__SELF__")) split_at = len(defaults) self._combine_default_lists(defaults, job_defaults) consumed = self._apply_overrides_to_defaults(overrides, defaults) # Load and defaults and merge them into cfg cfg = self._merge_defaults_into_config(hydra_cfg, job_cfg, job_cfg_load_trace, defaults, split_at) OmegaConf.set_struct(cfg.hydra, True) OmegaConf.set_struct(cfg, strict) # Merge all command line overrides after enabling strict flag remaining_overrides = [x for x in overrides if x not in consumed] try: merged = OmegaConf.merge( cfg, OmegaConf.from_dotlist(remaining_overrides)) assert isinstance(merged, DictConfig) cfg = merged except OmegaConfBaseException as ex: raise HydraException("Error merging overrides") from ex remaining = consumed + remaining_overrides def is_hydra(x: str) -> bool: return x.startswith("hydra.") or x.startswith("hydra/") cfg.hydra.overrides.task = [x for x in remaining if not is_hydra(x)] cfg.hydra.overrides.hydra = [x for x in remaining if is_hydra(x)] with open_dict(cfg.hydra.job): if "name" not in cfg.hydra.job: cfg.hydra.job.name = JobRuntime().get("name") cfg.hydra.job.override_dirname = get_overrides_dirname( input_list=cfg.hydra.overrides.task, kv_sep=cfg.hydra.job.config.override_dirname.kv_sep, item_sep=cfg.hydra.job.config.override_dirname.item_sep, exclude_keys=cfg.hydra.job.config.override_dirname. exclude_keys, ) cfg.hydra.job.config_name = config_name for key in cfg.hydra.job.env_copy: cfg.hydra.job.env_set[key] = os.environ[key] return cfg