def update_package_header(repo: IConfigRepository, node: InputDefault) -> None: if node.is_missing(): return # This loads the same config loaded in _create_defaults_tree # To avoid loading it twice, the repo implementation is expected to cache loaded configs loaded = repo.load_config(config_path=node.get_config_path()) if loaded is not None: node.set_package_header(loaded.header["package"])
def _compute_element_defaults_list_impl( element: DefaultElement, group_to_choice: DictConfig, delete_groups: Dict[DeleteKey, int], skip_missing: bool, repo: IConfigRepository, ) -> List[DefaultElement]: deleted = delete_if_matching(delete_groups, element) if deleted: return [] if element.config_name == "???": if skip_missing: element.set_skip_load("missing_skipped") return [element] else: if element.config_group is not None: options = repo.get_group_options( element.config_group, results_filter=ObjectType.CONFIG ) opt_list = "\n".join(["\t" + x for x in options]) msg = ( f"You must specify '{element.config_group}', e.g, {element.config_group}=<OPTION>" f"\nAvailable options:" f"\n{opt_list}" ) else: msg = f"You must specify '{element.config_group}', e.g, {element.config_group}=<OPTION>" raise ConfigCompositionException(msg) loaded = repo.load_config( config_path=element.config_path(), is_primary_config=element.primary, ) if loaded is None: if element.optional: element.set_skip_load("missing_optional_config") return [element] else: missing_config_error(repo=repo, element=element) else: original = copy.deepcopy(loaded.defaults_list) effective = copy.deepcopy(loaded.defaults_list) defaults = DefaultsList(original=original, effective=effective) _validate_self(element, defaults) return _expand_defaults_list_impl( self_element=element, defaults_list=defaults, group_to_choice=group_to_choice, delete_groups=delete_groups, skip_missing=skip_missing, repo=repo, )
def _load_single_config(self, default: ResultDefault, repo: IConfigRepository) -> ConfigResult: config_path = default.config_path assert config_path is not None ret = repo.load_config(config_path=config_path) assert ret is not None if not OmegaConf.is_config(ret.config): raise ValueError( f"Config {config_path} must be an OmegaConf config, got {type(ret.config).__name__}" ) if not ret.is_schema_source: schema = None try: schema_source = repo.get_schema_source() cname = ConfigSource._normalize_file_name(filename=config_path) schema = schema_source.load_config(cname) except ConfigLoadError: # schema not found, ignore pass if schema is not None: try: url = "https://hydra.cc/docs/next/upgrades/1.0_to_1.1/automatic_schema_matching" if "defaults" in schema.config: raise ConfigCompositionException( dedent(f"""\ '{config_path}' is validated against ConfigStore schema with the same name. This behavior is deprecated in Hydra 1.1 and will be removed in Hydra 1.2. In addition, the automatically matched schema contains a defaults list. This combination is no longer supported. See {url} for migration instructions.""")) else: warnings.warn( dedent(f"""\ '{config_path}' is validated against ConfigStore schema with the same name. This behavior is deprecated in Hydra 1.1 and will be removed in Hydra 1.2. See {url} for migration instructions."""), category=UserWarning, stacklevel=11, ) # if primary config has a hydra node, remove it during validation and add it back. # This allows overriding Hydra's configuration without declaring it's node # in the schema of every primary config hydra = None hydra_config_group = ( default.config_path is not None and default.config_path.startswith("hydra/")) config = ret.config if (default.primary and isinstance(config, DictConfig) and "hydra" in config and not hydra_config_group): hydra = config.pop("hydra") merged = OmegaConf.merge(schema.config, config) assert isinstance(merged, DictConfig) if hydra is not None: with open_dict(merged): merged.hydra = hydra ret.config = merged except OmegaConfBaseException as e: raise ConfigCompositionException( f"Error merging '{config_path}' with schema") from e assert isinstance(merged, DictConfig) res = self._embed_result_config(ret, default.package) if (not default.primary and config_path != "hydra/config" and isinstance(res.config, DictConfig) and OmegaConf.select( res.config, "hydra.searchpath") is not None): raise ConfigCompositionException( f"In '{config_path}': Overriding hydra.searchpath is only supported from the primary config" ) return res
def _create_defaults_tree_impl( repo: IConfigRepository, root: DefaultsTreeNode, is_root_config: bool, skip_missing: bool, interpolated_subtree: bool, overrides: Overrides, ) -> DefaultsTreeNode: parent = root.node children: List[Union[InputDefault, DefaultsTreeNode]] = [] if parent.is_virtual(): if is_root_config: return _expand_virtual_root(repo, root, overrides, skip_missing) else: return root if is_root_config: root.node.update_parent("", "") if not repo.config_exists(root.node.get_config_path()): config_not_found_error(repo=repo, tree=root) update_package_header(repo=repo, node=parent) if overrides.is_deleted(parent): overrides.delete(parent) return root overrides.set_known_choice(parent) if parent.get_name() is None: return root if _check_not_missing(repo=repo, default=parent, skip_missing=skip_missing): return root path = parent.get_config_path() loaded = repo.load_config(config_path=path) if loaded is None: if parent.is_optional(): assert isinstance(parent, (GroupDefault, ConfigDefault)) parent.deleted = True return root config_not_found_error(repo=repo, tree=root) assert loaded is not None defaults_list = copy.deepcopy(loaded.defaults_list) if defaults_list is None: defaults_list = [] self_added = False if (len(defaults_list) > 0 or is_root_config and len(overrides.append_group_defaults) > 0): self_added = _validate_self(containing_node=parent, defaults=defaults_list) if is_root_config: # To ensure config overrides are last, insert the external overrides before the first config override. insert_idx = len(defaults_list) for idx, default in enumerate(defaults_list): if default.is_override(): insert_idx = idx break defaults_list[insert_idx:insert_idx] = overrides.append_group_defaults _update_overrides(defaults_list, overrides, parent, interpolated_subtree) def add_child( child_list: List[Union[InputDefault, DefaultsTreeNode]], new_root_: DefaultsTreeNode, ) -> None: subtree_ = _create_defaults_tree_impl( repo=repo, root=new_root_, is_root_config=False, interpolated_subtree=interpolated_subtree, skip_missing=skip_missing, overrides=overrides, ) if subtree_.children is None: child_list.append(new_root_.node) else: child_list.append(subtree_) for d in reversed(defaults_list): if d.is_self(): d.update_parent(root.node.parent_base_dir, root.node.get_package()) children.append(d) else: if d.is_override(): continue d.update_parent(parent.get_group_path(), parent.get_final_package()) if overrides.is_overridden(d): assert isinstance(d, GroupDefault) overrides.override_default_option(d) if isinstance(d, GroupDefault) and d.is_options(): # overriding may change from options to name if d.is_options(): for item in reversed(d.get_options()): if "${" in item: raise ConfigCompositionException( f"In '{path}': Defaults List interpolation is not supported in options list items" ) assert d.group is not None node = ConfigDefault( path=d.group + "/" + item, package=d.package, optional=d.is_optional(), ) node.update_parent(parent.get_group_path(), parent.get_final_package()) new_root = DefaultsTreeNode(node=node, parent=root) add_child(children, new_root) else: new_root = DefaultsTreeNode(node=d, parent=root) add_child(children, new_root) else: if d.is_interpolation(): children.append(d) continue new_root = DefaultsTreeNode(node=d, parent=root) add_child(children, new_root) # processed deferred interpolations known_choices = _create_interpolation_map(overrides, defaults_list, self_added) for idx, dd in enumerate(children): if isinstance(dd, InputDefault) and dd.is_interpolation(): dd.resolve_interpolation(known_choices) new_root = DefaultsTreeNode(node=dd, parent=root) dd.update_parent(parent.get_group_path(), parent.get_final_package()) subtree = _create_defaults_tree_impl( repo=repo, root=new_root, is_root_config=False, skip_missing=skip_missing, interpolated_subtree=True, overrides=overrides, ) if subtree.children is not None: children[idx] = subtree if len(children) > 0: root.children = list(reversed(children)) return root
def _load_single_config( self, default: DefaultElement, repo: IConfigRepository, ) -> Tuple[ConfigResult, LoadTrace]: config_path = default.config_path() package_override = default.package ret = repo.load_config( config_path=config_path, is_primary_config=default.primary, package_override=package_override, ) assert ret is not None if not isinstance(ret.config, DictConfig): raise ValueError( f"Config {config_path} must be a Dictionary, got {type(ret).__name__}" ) default.search_path = ret.path schema_provider = None if not ret.is_schema_source: schema = None try: schema_source = repo.get_schema_source() schema = schema_source.load_config( ConfigSource._normalize_file_name(filename=config_path), is_primary_config=default.primary, package_override=package_override, ) except ConfigLoadError: # schema not found, ignore pass if schema is not None: try: # if config has a hydra node, remove it during validation and add it back. # This allows overriding Hydra's configuration without declaring this node # in every program hydra = None hydra_config_group = ( default.config_group is not None and default.config_group.startswith("hydra/")) if "hydra" in ret.config and not hydra_config_group: hydra = ret.config.pop("hydra") schema_provider = schema.provider merged = OmegaConf.merge(schema.config, ret.config) assert isinstance(merged, DictConfig) if hydra is not None: with open_dict(merged): merged.hydra = hydra ret.config = merged except OmegaConfBaseException as e: raise ConfigCompositionException( f"Error merging '{config_path}' with schema") from e assert isinstance(merged, DictConfig) trace = LoadTrace( config_group=default.config_group, config_name=default.config_name, package=default.get_subject_package(), search_path=ret.path, parent=default.parent, provider=ret.provider, schema_provider=schema_provider, ) return ret, trace
def _create_defaults_tree_impl( repo: IConfigRepository, root: DefaultsTreeNode, is_root_config: bool, skip_missing: bool, interpolated_subtree: bool, overrides: Overrides, ) -> DefaultsTreeNode: parent = root.node children: List[Union[InputDefault, DefaultsTreeNode]] = [] if parent.is_virtual(): if is_root_config: return _expand_virtual_root(repo, root, overrides, skip_missing) else: return root else: if is_root_config: root.node.update_parent("", "") if is_root_config: if not repo.config_exists(root.node.get_config_path()): config_not_found_error(repo=repo, tree=root) update_package_header(repo=repo, node=parent) if overrides.is_overridden(parent): assert isinstance(parent, GroupDefault) overrides.override_default_option(parent) # clear package header and obtain updated one from overridden config # (for the rare case it has changed) parent.package_header = None update_package_header(repo=repo, node=parent) if overrides.is_deleted(parent): overrides.delete(parent) return root overrides.set_known_choice(parent) if parent.get_name() is None: return root if _check_not_missing(repo=repo, default=parent, skip_missing=skip_missing): return root path = parent.get_config_path() loaded = repo.load_config(config_path=path) if loaded is None: if parent.is_optional(): assert isinstance(parent, GroupDefault) parent.deleted = True return root config_not_found_error(repo=repo, tree=root) assert loaded is not None defaults_list = copy.deepcopy(loaded.defaults_list) if defaults_list is None: defaults_list = [] self_added = False if (len(defaults_list) > 0 or is_root_config and len(overrides.append_group_defaults) > 0): self_added = _validate_self(containing_node=parent, defaults=defaults_list) if is_root_config: defaults_list.extend(overrides.append_group_defaults) _update_overrides(defaults_list, overrides, parent, interpolated_subtree) for d in reversed(defaults_list): if d.is_self(): d.update_parent(root.node.parent_base_dir, root.node.get_package()) children.append(d) else: if d.is_override(): continue new_root = DefaultsTreeNode(node=d, parent=root) d.update_parent(parent.get_group_path(), parent.get_final_package()) if d.is_interpolation(): children.append(d) continue subtree = _create_defaults_tree_impl( repo=repo, root=new_root, is_root_config=False, interpolated_subtree=interpolated_subtree, skip_missing=skip_missing, overrides=overrides, ) if subtree.children is None: children.append(d) else: children.append(subtree) # processed deferred interpolations known_choices = _create_interpolation_map(overrides, defaults_list, self_added) for idx, dd in enumerate(children): if isinstance(dd, InputDefault) and dd.is_interpolation(): if not parent.primary: # Interpolations from nested configs would require much more work # If you have a compelling use case please file an feature request. path = parent.get_config_path() raise ConfigCompositionException( f"In '{path}': Defaults List interpolation is only supported in the primary config" ) dd.resolve_interpolation(known_choices) new_root = DefaultsTreeNode(node=dd, parent=root) dd.update_parent(parent.get_group_path(), parent.get_final_package()) subtree = _create_defaults_tree_impl( repo=repo, root=new_root, is_root_config=False, skip_missing=skip_missing, interpolated_subtree=True, overrides=overrides, ) if subtree.children is not None: children[idx] = subtree if len(children) > 0: root.children = list(reversed(children)) return root
def _load_single_config( self, default: ResultDefault, repo: IConfigRepository ) -> Tuple[ConfigResult, LoadTrace]: config_path = default.config_path assert config_path is not None ret = repo.load_config(config_path=config_path) assert ret is not None if not isinstance(ret.config, DictConfig): raise ValueError( f"Config {config_path} must be a Dictionary, got {type(ret).__name__}" ) if not ret.is_schema_source: schema = None try: schema_source = repo.get_schema_source() cname = ConfigSource._normalize_file_name(filename=config_path) schema = schema_source.load_config(cname) except ConfigLoadError: # schema not found, ignore pass if schema is not None: try: # TODO: deprecate schema matching in favor of extension via Defaults List # if primary config has a hydra node, remove it during validation and add it back. # This allows overriding Hydra's configuration without declaring it's node # in the schema of every primary config hydra = None hydra_config_group = ( default.config_path is not None and default.config_path.startswith("hydra/") ) if ( default.primary and "hydra" in ret.config and not hydra_config_group ): hydra = ret.config.pop("hydra") merged = OmegaConf.merge(schema.config, ret.config) assert isinstance(merged, DictConfig) if hydra is not None: with open_dict(merged): merged.hydra = hydra ret.config = merged except OmegaConfBaseException as e: raise ConfigCompositionException( f"Error merging '{config_path}' with schema" ) from e assert isinstance(merged, DictConfig) trace = LoadTrace( config_path=default.config_path, package=default.package, parent=default.parent, is_self=default.is_self, search_path=ret.path, provider=ret.provider, ) ret = self._embed_result_config(ret, default.package) return ret, trace