def missing_config_error(repo: IConfigRepository, element: DefaultElement) -> None: options = None if element.config_group is not None: options = repo.get_group_options(element.config_group, ObjectType.CONFIG) opt_list = "\n".join(["\t" + x for x in options]) msg = ( f"Could not find '{element.config_name}' in the config group '{element.config_group}'" f"\nAvailable options:\n{opt_list}\n" ) else: msg = dedent( f"""\ Could not load {element.config_path()}. """ ) descs = [] for src in repo.get_sources(): descs.append(f"\t{repr(src)}") lines = "\n".join(descs) msg += "\nConfig search path:" + f"\n{lines}" raise MissingConfigException( missing_cfg_file=element.config_path(), message=msg, options=options, )
def _compute_element_defaults_list_impl( element: DefaultElement, group_to_choice: DictConfig, delete_groups: Dict[DeleteKey, int], skip_missing: bool, repo: IConfigRepository, ) -> List[DefaultElement]: deleted = delete_if_matching(delete_groups, element) if deleted: return [] if element.config_name == "???": if skip_missing: element.set_skip_load("missing_skipped") return [element] else: if element.config_group is not None: options = repo.get_group_options( element.config_group, results_filter=ObjectType.CONFIG ) opt_list = "\n".join(["\t" + x for x in options]) msg = ( f"You must specify '{element.config_group}', e.g, {element.config_group}=<OPTION>" f"\nAvailable options:" f"\n{opt_list}" ) else: msg = f"You must specify '{element.config_group}', e.g, {element.config_group}=<OPTION>" raise ConfigCompositionException(msg) loaded = repo.load_config( config_path=element.config_path(), is_primary_config=element.primary, ) if loaded is None: if element.optional: element.set_skip_load("missing_optional_config") return [element] else: missing_config_error(repo=repo, element=element) else: original = copy.deepcopy(loaded.defaults_list) effective = copy.deepcopy(loaded.defaults_list) defaults = DefaultsList(original=original, effective=effective) _validate_self(element, defaults) return _expand_defaults_list_impl( self_element=element, defaults_list=defaults, group_to_choice=group_to_choice, delete_groups=delete_groups, skip_missing=skip_missing, repo=repo, )
def _check_not_missing( repo: IConfigRepository, default: InputDefault, skip_missing: bool, ) -> bool: path = default.get_config_path() if path.endswith("???"): if skip_missing: return True if isinstance(default, GroupDefault): group_path = default.get_group_path() options = repo.get_group_options( group_path, results_filter=ObjectType.CONFIG, ) opt_list = "\n".join(["\t" + x for x in options]) msg = dedent(f"""\ You must specify '{group_path}', e.g, {group_path}=<OPTION> Available options: """) raise ConfigCompositionException(msg + opt_list) elif isinstance(default, ConfigDefault): raise ValueError( f"Missing ConfigDefault is not supported : {path}") else: assert False return False
def update_package_header(repo: IConfigRepository, node: InputDefault) -> None: if node.is_missing(): return # This loads the same config loaded in _create_defaults_tree # To avoid loading it twice, the repo implementation is expected to cache loaded configs loaded = repo.load_config(config_path=node.get_config_path()) if loaded is not None: node.set_package_header(loaded.header["package"])
def config_not_found_error(repo: IConfigRepository, tree: DefaultsTreeNode) -> None: element = tree.node options = None group = None if isinstance(element, GroupDefault): group = element.get_group_path() options = repo.get_group_options(group, ObjectType.CONFIG) if element.primary: msg = dedent( f"""\ Cannot find primary config '{element.get_config_path()}'. Check that it's in your config search path. """ ) else: parent = tree.parent.node if tree.parent is not None else None if isinstance(element, GroupDefault): msg = f"Could not find '{element.get_config_path()}'\n" if options is not None and len(options) > 0: opt_list = "\n".join(["\t" + x for x in options]) msg = f"{msg}\nAvailable options in '{group}':\n" + opt_list else: msg = dedent( f"""\ Could not load '{element.get_config_path()}'. """ ) if parent is not None: msg = f"In '{parent.get_config_path()}': {msg}" descs = [] for src in repo.get_sources(): descs.append(f"\t{repr(src)}") lines = "\n".join(descs) msg += "\nConfig search path:" + f"\n{lines}" raise MissingConfigException( missing_cfg_file=element.get_config_path(), message=msg, options=options, )
def __init__(self, repo: IConfigRepository, overrides_list: List[Override]) -> None: self.override_choices = {} self.override_metadata = {} self.append_group_defaults = [] self.config_overrides = [] self.deletions = {} self.known_choices = {} self.known_choices_per_group = {} for override in overrides_list: if override.is_sweep_override(): continue is_group = repo.group_exists(override.key_or_group) value = override.value() is_dict = isinstance(override.value(), dict) if is_dict or not is_group: self.config_overrides.append(override) elif override.is_force_add(): # This could probably be made to work if there is a compelling use case. raise ConfigCompositionException( f"force-add of config groups is not supported: '{override.input_line}'" ) elif override.is_delete(): key = override.get_key_element()[1:] value = override.value() if value is not None and not isinstance(value, str): raise ValueError( f"Config group override deletion value must be a string : {override}" ) self.deletions[key] = Deletion(name=value) elif not isinstance(value, (str, list)): raise ValueError( f"Config group override must be a string or a list. Got {type(value).__name__}" ) elif override.is_add(): self.append_group_defaults.append( GroupDefault( group=override.key_or_group, package=override.package, value=value, external_append=True, )) else: key = override.get_key_element() self.override_choices[key] = value self.override_metadata[key] = OverrideMetadata( external_override=True)
def __init__(self, repo: IConfigRepository, overrides_list: List[Override]) -> None: self.override_choices = {} self.override_metadata = {} self.append_group_defaults = [] self.config_overrides = [] self.deletions = {} self.known_choices = {} self.known_choices_per_group = {} for override in overrides_list: if override.is_sweep_override(): continue is_group = repo.group_exists(override.key_or_group) value = override.value() is_dict = isinstance(override.value(), dict) if is_dict or not is_group: self.config_overrides.append(override) else: if override.is_delete(): key = override.get_key_element()[1:] value = override.value() if value is not None and not isinstance(value, str): raise ValueError( f"Config group override deletion value must be a string : {override}" ) self.deletions[key] = Deletion(name=value) elif not isinstance(value, (str, list)): raise ValueError( f"Config group override must be a string or a list. Got {type(value).__name__}" ) elif override.is_add(): self.append_group_defaults.append( GroupDefault( group=override.key_or_group, package=override.package, value=value, ) ) else: key = override.get_key_element() self.override_choices[key] = value self.override_metadata[key] = OverrideMetadata( external_override=True )
def _load_single_config(self, default: ResultDefault, repo: IConfigRepository) -> ConfigResult: config_path = default.config_path assert config_path is not None ret = repo.load_config(config_path=config_path) assert ret is not None if not OmegaConf.is_config(ret.config): raise ValueError( f"Config {config_path} must be an OmegaConf config, got {type(ret.config).__name__}" ) if not ret.is_schema_source: schema = None try: schema_source = repo.get_schema_source() cname = ConfigSource._normalize_file_name(filename=config_path) schema = schema_source.load_config(cname) except ConfigLoadError: # schema not found, ignore pass if schema is not None: try: url = "https://hydra.cc/docs/next/upgrades/1.0_to_1.1/automatic_schema_matching" if "defaults" in schema.config: raise ConfigCompositionException( dedent(f"""\ '{config_path}' is validated against ConfigStore schema with the same name. This behavior is deprecated in Hydra 1.1 and will be removed in Hydra 1.2. In addition, the automatically matched schema contains a defaults list. This combination is no longer supported. See {url} for migration instructions.""")) else: warnings.warn( dedent(f"""\ '{config_path}' is validated against ConfigStore schema with the same name. This behavior is deprecated in Hydra 1.1 and will be removed in Hydra 1.2. See {url} for migration instructions."""), category=UserWarning, stacklevel=11, ) # if primary config has a hydra node, remove it during validation and add it back. # This allows overriding Hydra's configuration without declaring it's node # in the schema of every primary config hydra = None hydra_config_group = ( default.config_path is not None and default.config_path.startswith("hydra/")) config = ret.config if (default.primary and isinstance(config, DictConfig) and "hydra" in config and not hydra_config_group): hydra = config.pop("hydra") merged = OmegaConf.merge(schema.config, config) assert isinstance(merged, DictConfig) if hydra is not None: with open_dict(merged): merged.hydra = hydra ret.config = merged except OmegaConfBaseException as e: raise ConfigCompositionException( f"Error merging '{config_path}' with schema") from e assert isinstance(merged, DictConfig) res = self._embed_result_config(ret, default.package) if (not default.primary and config_path != "hydra/config" and isinstance(res.config, DictConfig) and OmegaConf.select( res.config, "hydra.searchpath") is not None): raise ConfigCompositionException( f"In '{config_path}': Overriding hydra.searchpath is only supported from the primary config" ) return res
def _create_defaults_tree_impl( repo: IConfigRepository, root: DefaultsTreeNode, is_root_config: bool, skip_missing: bool, interpolated_subtree: bool, overrides: Overrides, ) -> DefaultsTreeNode: parent = root.node children: List[Union[InputDefault, DefaultsTreeNode]] = [] if parent.is_virtual(): if is_root_config: return _expand_virtual_root(repo, root, overrides, skip_missing) else: return root if is_root_config: root.node.update_parent("", "") if not repo.config_exists(root.node.get_config_path()): config_not_found_error(repo=repo, tree=root) update_package_header(repo=repo, node=parent) if overrides.is_deleted(parent): overrides.delete(parent) return root overrides.set_known_choice(parent) if parent.get_name() is None: return root if _check_not_missing(repo=repo, default=parent, skip_missing=skip_missing): return root path = parent.get_config_path() loaded = repo.load_config(config_path=path) if loaded is None: if parent.is_optional(): assert isinstance(parent, (GroupDefault, ConfigDefault)) parent.deleted = True return root config_not_found_error(repo=repo, tree=root) assert loaded is not None defaults_list = copy.deepcopy(loaded.defaults_list) if defaults_list is None: defaults_list = [] self_added = False if (len(defaults_list) > 0 or is_root_config and len(overrides.append_group_defaults) > 0): self_added = _validate_self(containing_node=parent, defaults=defaults_list) if is_root_config: # To ensure config overrides are last, insert the external overrides before the first config override. insert_idx = len(defaults_list) for idx, default in enumerate(defaults_list): if default.is_override(): insert_idx = idx break defaults_list[insert_idx:insert_idx] = overrides.append_group_defaults _update_overrides(defaults_list, overrides, parent, interpolated_subtree) def add_child( child_list: List[Union[InputDefault, DefaultsTreeNode]], new_root_: DefaultsTreeNode, ) -> None: subtree_ = _create_defaults_tree_impl( repo=repo, root=new_root_, is_root_config=False, interpolated_subtree=interpolated_subtree, skip_missing=skip_missing, overrides=overrides, ) if subtree_.children is None: child_list.append(new_root_.node) else: child_list.append(subtree_) for d in reversed(defaults_list): if d.is_self(): d.update_parent(root.node.parent_base_dir, root.node.get_package()) children.append(d) else: if d.is_override(): continue d.update_parent(parent.get_group_path(), parent.get_final_package()) if overrides.is_overridden(d): assert isinstance(d, GroupDefault) overrides.override_default_option(d) if isinstance(d, GroupDefault) and d.is_options(): # overriding may change from options to name if d.is_options(): for item in reversed(d.get_options()): if "${" in item: raise ConfigCompositionException( f"In '{path}': Defaults List interpolation is not supported in options list items" ) assert d.group is not None node = ConfigDefault( path=d.group + "/" + item, package=d.package, optional=d.is_optional(), ) node.update_parent(parent.get_group_path(), parent.get_final_package()) new_root = DefaultsTreeNode(node=node, parent=root) add_child(children, new_root) else: new_root = DefaultsTreeNode(node=d, parent=root) add_child(children, new_root) else: if d.is_interpolation(): children.append(d) continue new_root = DefaultsTreeNode(node=d, parent=root) add_child(children, new_root) # processed deferred interpolations known_choices = _create_interpolation_map(overrides, defaults_list, self_added) for idx, dd in enumerate(children): if isinstance(dd, InputDefault) and dd.is_interpolation(): dd.resolve_interpolation(known_choices) new_root = DefaultsTreeNode(node=dd, parent=root) dd.update_parent(parent.get_group_path(), parent.get_final_package()) subtree = _create_defaults_tree_impl( repo=repo, root=new_root, is_root_config=False, skip_missing=skip_missing, interpolated_subtree=True, overrides=overrides, ) if subtree.children is not None: children[idx] = subtree if len(children) > 0: root.children = list(reversed(children)) return root
def _load_single_config( self, default: DefaultElement, repo: IConfigRepository, ) -> Tuple[ConfigResult, LoadTrace]: config_path = default.config_path() package_override = default.package ret = repo.load_config( config_path=config_path, is_primary_config=default.primary, package_override=package_override, ) assert ret is not None if not isinstance(ret.config, DictConfig): raise ValueError( f"Config {config_path} must be a Dictionary, got {type(ret).__name__}" ) default.search_path = ret.path schema_provider = None if not ret.is_schema_source: schema = None try: schema_source = repo.get_schema_source() schema = schema_source.load_config( ConfigSource._normalize_file_name(filename=config_path), is_primary_config=default.primary, package_override=package_override, ) except ConfigLoadError: # schema not found, ignore pass if schema is not None: try: # if config has a hydra node, remove it during validation and add it back. # This allows overriding Hydra's configuration without declaring this node # in every program hydra = None hydra_config_group = ( default.config_group is not None and default.config_group.startswith("hydra/")) if "hydra" in ret.config and not hydra_config_group: hydra = ret.config.pop("hydra") schema_provider = schema.provider merged = OmegaConf.merge(schema.config, ret.config) assert isinstance(merged, DictConfig) if hydra is not None: with open_dict(merged): merged.hydra = hydra ret.config = merged except OmegaConfBaseException as e: raise ConfigCompositionException( f"Error merging '{config_path}' with schema") from e assert isinstance(merged, DictConfig) trace = LoadTrace( config_group=default.config_group, config_name=default.config_name, package=default.get_subject_package(), search_path=ret.path, parent=default.parent, provider=ret.provider, schema_provider=schema_provider, ) return ret, trace
def _create_defaults_tree_impl( repo: IConfigRepository, root: DefaultsTreeNode, is_root_config: bool, skip_missing: bool, interpolated_subtree: bool, overrides: Overrides, ) -> DefaultsTreeNode: parent = root.node children: List[Union[InputDefault, DefaultsTreeNode]] = [] if parent.is_virtual(): if is_root_config: return _expand_virtual_root(repo, root, overrides, skip_missing) else: return root else: if is_root_config: root.node.update_parent("", "") if is_root_config: if not repo.config_exists(root.node.get_config_path()): config_not_found_error(repo=repo, tree=root) update_package_header(repo=repo, node=parent) if overrides.is_overridden(parent): assert isinstance(parent, GroupDefault) overrides.override_default_option(parent) # clear package header and obtain updated one from overridden config # (for the rare case it has changed) parent.package_header = None update_package_header(repo=repo, node=parent) if overrides.is_deleted(parent): overrides.delete(parent) return root overrides.set_known_choice(parent) if parent.get_name() is None: return root if _check_not_missing(repo=repo, default=parent, skip_missing=skip_missing): return root path = parent.get_config_path() loaded = repo.load_config(config_path=path) if loaded is None: if parent.is_optional(): assert isinstance(parent, GroupDefault) parent.deleted = True return root config_not_found_error(repo=repo, tree=root) assert loaded is not None defaults_list = copy.deepcopy(loaded.defaults_list) if defaults_list is None: defaults_list = [] self_added = False if (len(defaults_list) > 0 or is_root_config and len(overrides.append_group_defaults) > 0): self_added = _validate_self(containing_node=parent, defaults=defaults_list) if is_root_config: defaults_list.extend(overrides.append_group_defaults) _update_overrides(defaults_list, overrides, parent, interpolated_subtree) for d in reversed(defaults_list): if d.is_self(): d.update_parent(root.node.parent_base_dir, root.node.get_package()) children.append(d) else: if d.is_override(): continue new_root = DefaultsTreeNode(node=d, parent=root) d.update_parent(parent.get_group_path(), parent.get_final_package()) if d.is_interpolation(): children.append(d) continue subtree = _create_defaults_tree_impl( repo=repo, root=new_root, is_root_config=False, interpolated_subtree=interpolated_subtree, skip_missing=skip_missing, overrides=overrides, ) if subtree.children is None: children.append(d) else: children.append(subtree) # processed deferred interpolations known_choices = _create_interpolation_map(overrides, defaults_list, self_added) for idx, dd in enumerate(children): if isinstance(dd, InputDefault) and dd.is_interpolation(): if not parent.primary: # Interpolations from nested configs would require much more work # If you have a compelling use case please file an feature request. path = parent.get_config_path() raise ConfigCompositionException( f"In '{path}': Defaults List interpolation is only supported in the primary config" ) dd.resolve_interpolation(known_choices) new_root = DefaultsTreeNode(node=dd, parent=root) dd.update_parent(parent.get_group_path(), parent.get_final_package()) subtree = _create_defaults_tree_impl( repo=repo, root=new_root, is_root_config=False, skip_missing=skip_missing, interpolated_subtree=True, overrides=overrides, ) if subtree.children is not None: children[idx] = subtree if len(children) > 0: root.children = list(reversed(children)) return root
def _load_single_config( self, default: ResultDefault, repo: IConfigRepository ) -> Tuple[ConfigResult, LoadTrace]: config_path = default.config_path assert config_path is not None ret = repo.load_config(config_path=config_path) assert ret is not None if not isinstance(ret.config, DictConfig): raise ValueError( f"Config {config_path} must be a Dictionary, got {type(ret).__name__}" ) if not ret.is_schema_source: schema = None try: schema_source = repo.get_schema_source() cname = ConfigSource._normalize_file_name(filename=config_path) schema = schema_source.load_config(cname) except ConfigLoadError: # schema not found, ignore pass if schema is not None: try: # TODO: deprecate schema matching in favor of extension via Defaults List # if primary config has a hydra node, remove it during validation and add it back. # This allows overriding Hydra's configuration without declaring it's node # in the schema of every primary config hydra = None hydra_config_group = ( default.config_path is not None and default.config_path.startswith("hydra/") ) if ( default.primary and "hydra" in ret.config and not hydra_config_group ): hydra = ret.config.pop("hydra") merged = OmegaConf.merge(schema.config, ret.config) assert isinstance(merged, DictConfig) if hydra is not None: with open_dict(merged): merged.hydra = hydra ret.config = merged except OmegaConfBaseException as e: raise ConfigCompositionException( f"Error merging '{config_path}' with schema" ) from e assert isinstance(merged, DictConfig) trace = LoadTrace( config_path=default.config_path, package=default.package, parent=default.parent, is_self=default.is_self, search_path=ret.path, provider=ret.provider, ) ret = self._embed_result_config(ret, default.package) return ret, trace