def missing_config_error(repo: IConfigRepository, element: DefaultElement) -> None:
options = None
if element.config_group is not None:
options = repo.get_group_options(element.config_group, ObjectType.CONFIG)
opt_list = "\n".join(["\t" + x for x in options])
msg = (
f"Could not find '{element.config_name}' in the config group '{element.config_group}'"
f"\nAvailable options:\n{opt_list}\n"
)
else:
msg = dedent(
f"""\
Could not load {element.config_path()}.
"""
)
descs = []
for src in repo.get_sources():
descs.append(f"\t{repr(src)}")
lines = "\n".join(descs)
msg += "\nConfig search path:" + f"\n{lines}"
raise MissingConfigException(
missing_cfg_file=element.config_path(),
message=msg,
options=options,
)
def _compute_element_defaults_list_impl(
element: DefaultElement,
group_to_choice: DictConfig,
delete_groups: Dict[DeleteKey, int],
skip_missing: bool,
repo: IConfigRepository,
) -> List[DefaultElement]:
deleted = delete_if_matching(delete_groups, element)
if deleted:
return []
if element.config_name == "???":
if skip_missing:
element.set_skip_load("missing_skipped")
return [element]
else:
if element.config_group is not None:
options = repo.get_group_options(
element.config_group, results_filter=ObjectType.CONFIG
)
opt_list = "\n".join(["\t" + x for x in options])
msg = (
f"You must specify '{element.config_group}', e.g, {element.config_group}=<OPTION>"
f"\nAvailable options:"
f"\n{opt_list}"
)
else:
msg = f"You must specify '{element.config_group}', e.g, {element.config_group}=<OPTION>"
raise ConfigCompositionException(msg)
loaded = repo.load_config(
config_path=element.config_path(),
is_primary_config=element.primary,
)
if loaded is None:
if element.optional:
element.set_skip_load("missing_optional_config")
return [element]
else:
missing_config_error(repo=repo, element=element)
else:
original = copy.deepcopy(loaded.defaults_list)
effective = copy.deepcopy(loaded.defaults_list)
defaults = DefaultsList(original=original, effective=effective)
_validate_self(element, defaults)
return _expand_defaults_list_impl(
self_element=element,
defaults_list=defaults,
group_to_choice=group_to_choice,
delete_groups=delete_groups,
skip_missing=skip_missing,
repo=repo,
)
def _check_not_missing(
repo: IConfigRepository,
default: InputDefault,
skip_missing: bool,
) -> bool:
path = default.get_config_path()
if path.endswith("???"):
if skip_missing:
return True
if isinstance(default, GroupDefault):
group_path = default.get_group_path()
options = repo.get_group_options(
group_path,
results_filter=ObjectType.CONFIG,
)
opt_list = "\n".join(["\t" + x for x in options])
msg = dedent(f"""\
You must specify '{group_path}', e.g, {group_path}=<OPTION>
Available options:
""")
raise ConfigCompositionException(msg + opt_list)
elif isinstance(default, ConfigDefault):
raise ValueError(
f"Missing ConfigDefault is not supported : {path}")
else:
assert False
return False
def update_package_header(repo: IConfigRepository, node: InputDefault) -> None:
if node.is_missing():
return
# This loads the same config loaded in _create_defaults_tree
# To avoid loading it twice, the repo implementation is expected to cache loaded configs
loaded = repo.load_config(config_path=node.get_config_path())
if loaded is not None:
node.set_package_header(loaded.header["package"])
def config_not_found_error(repo: IConfigRepository, tree: DefaultsTreeNode) -> None:
element = tree.node
options = None
group = None
if isinstance(element, GroupDefault):
group = element.get_group_path()
options = repo.get_group_options(group, ObjectType.CONFIG)
if element.primary:
msg = dedent(
f"""\
Cannot find primary config '{element.get_config_path()}'. Check that it's in your config search path.
"""
)
else:
parent = tree.parent.node if tree.parent is not None else None
if isinstance(element, GroupDefault):
msg = f"Could not find '{element.get_config_path()}'\n"
if options is not None and len(options) > 0:
opt_list = "\n".join(["\t" + x for x in options])
msg = f"{msg}\nAvailable options in '{group}':\n" + opt_list
else:
msg = dedent(
f"""\
Could not load '{element.get_config_path()}'.
"""
)
if parent is not None:
msg = f"In '{parent.get_config_path()}': {msg}"
descs = []
for src in repo.get_sources():
descs.append(f"\t{repr(src)}")
lines = "\n".join(descs)
msg += "\nConfig search path:" + f"\n{lines}"
raise MissingConfigException(
missing_cfg_file=element.get_config_path(),
message=msg,
options=options,
)
def __init__(self, repo: IConfigRepository,
overrides_list: List[Override]) -> None:
self.override_choices = {}
self.override_metadata = {}
self.append_group_defaults = []
self.config_overrides = []
self.deletions = {}
self.known_choices = {}
self.known_choices_per_group = {}
for override in overrides_list:
if override.is_sweep_override():
continue
is_group = repo.group_exists(override.key_or_group)
value = override.value()
is_dict = isinstance(override.value(), dict)
if is_dict or not is_group:
self.config_overrides.append(override)
elif override.is_force_add():
# This could probably be made to work if there is a compelling use case.
raise ConfigCompositionException(
f"force-add of config groups is not supported: '{override.input_line}'"
)
elif override.is_delete():
key = override.get_key_element()[1:]
value = override.value()
if value is not None and not isinstance(value, str):
raise ValueError(
f"Config group override deletion value must be a string : {override}"
)
self.deletions[key] = Deletion(name=value)
elif not isinstance(value, (str, list)):
raise ValueError(
f"Config group override must be a string or a list. Got {type(value).__name__}"
)
elif override.is_add():
self.append_group_defaults.append(
GroupDefault(
group=override.key_or_group,
package=override.package,
value=value,
external_append=True,
))
else:
key = override.get_key_element()
self.override_choices[key] = value
self.override_metadata[key] = OverrideMetadata(
external_override=True)
def __init__(self, repo: IConfigRepository, overrides_list: List[Override]) -> None:
self.override_choices = {}
self.override_metadata = {}
self.append_group_defaults = []
self.config_overrides = []
self.deletions = {}
self.known_choices = {}
self.known_choices_per_group = {}
for override in overrides_list:
if override.is_sweep_override():
continue
is_group = repo.group_exists(override.key_or_group)
value = override.value()
is_dict = isinstance(override.value(), dict)
if is_dict or not is_group:
self.config_overrides.append(override)
else:
if override.is_delete():
key = override.get_key_element()[1:]
value = override.value()
if value is not None and not isinstance(value, str):
raise ValueError(
f"Config group override deletion value must be a string : {override}"
)
self.deletions[key] = Deletion(name=value)
elif not isinstance(value, (str, list)):
raise ValueError(
f"Config group override must be a string or a list. Got {type(value).__name__}"
)
elif override.is_add():
self.append_group_defaults.append(
GroupDefault(
group=override.key_or_group,
package=override.package,
value=value,
)
)
else:
key = override.get_key_element()
self.override_choices[key] = value
self.override_metadata[key] = OverrideMetadata(
external_override=True
)
def _load_single_config(self, default: ResultDefault,
repo: IConfigRepository) -> ConfigResult:
config_path = default.config_path
assert config_path is not None
ret = repo.load_config(config_path=config_path)
assert ret is not None
if not OmegaConf.is_config(ret.config):
raise ValueError(
f"Config {config_path} must be an OmegaConf config, got {type(ret.config).__name__}"
)
if not ret.is_schema_source:
schema = None
try:
schema_source = repo.get_schema_source()
cname = ConfigSource._normalize_file_name(filename=config_path)
schema = schema_source.load_config(cname)
except ConfigLoadError:
# schema not found, ignore
pass
if schema is not None:
try:
url = "https://hydra.cc/docs/next/upgrades/1.0_to_1.1/automatic_schema_matching"
if "defaults" in schema.config:
raise ConfigCompositionException(
dedent(f"""\
'{config_path}' is validated against ConfigStore schema with the same name.
This behavior is deprecated in Hydra 1.1 and will be removed in Hydra 1.2.
In addition, the automatically matched schema contains a defaults list.
This combination is no longer supported.
See {url} for migration instructions."""))
else:
warnings.warn(
dedent(f"""\
'{config_path}' is validated against ConfigStore schema with the same name.
This behavior is deprecated in Hydra 1.1 and will be removed in Hydra 1.2.
See {url} for migration instructions."""),
category=UserWarning,
stacklevel=11,
)
# if primary config has a hydra node, remove it during validation and add it back.
# This allows overriding Hydra's configuration without declaring it's node
# in the schema of every primary config
hydra = None
hydra_config_group = (
default.config_path is not None
and default.config_path.startswith("hydra/"))
config = ret.config
if (default.primary and isinstance(config, DictConfig)
and "hydra" in config and not hydra_config_group):
hydra = config.pop("hydra")
merged = OmegaConf.merge(schema.config, config)
assert isinstance(merged, DictConfig)
if hydra is not None:
with open_dict(merged):
merged.hydra = hydra
ret.config = merged
except OmegaConfBaseException as e:
raise ConfigCompositionException(
f"Error merging '{config_path}' with schema") from e
assert isinstance(merged, DictConfig)
res = self._embed_result_config(ret, default.package)
if (not default.primary and config_path != "hydra/config"
and isinstance(res.config, DictConfig) and OmegaConf.select(
res.config, "hydra.searchpath") is not None):
raise ConfigCompositionException(
f"In '{config_path}': Overriding hydra.searchpath is only supported from the primary config"
)
return res
def _create_defaults_tree_impl(
repo: IConfigRepository,
root: DefaultsTreeNode,
is_root_config: bool,
skip_missing: bool,
interpolated_subtree: bool,
overrides: Overrides,
) -> DefaultsTreeNode:
parent = root.node
children: List[Union[InputDefault, DefaultsTreeNode]] = []
if parent.is_virtual():
if is_root_config:
return _expand_virtual_root(repo, root, overrides, skip_missing)
else:
return root
if is_root_config:
root.node.update_parent("", "")
if not repo.config_exists(root.node.get_config_path()):
config_not_found_error(repo=repo, tree=root)
update_package_header(repo=repo, node=parent)
if overrides.is_deleted(parent):
overrides.delete(parent)
return root
overrides.set_known_choice(parent)
if parent.get_name() is None:
return root
if _check_not_missing(repo=repo, default=parent,
skip_missing=skip_missing):
return root
path = parent.get_config_path()
loaded = repo.load_config(config_path=path)
if loaded is None:
if parent.is_optional():
assert isinstance(parent, (GroupDefault, ConfigDefault))
parent.deleted = True
return root
config_not_found_error(repo=repo, tree=root)
assert loaded is not None
defaults_list = copy.deepcopy(loaded.defaults_list)
if defaults_list is None:
defaults_list = []
self_added = False
if (len(defaults_list) > 0
or is_root_config and len(overrides.append_group_defaults) > 0):
self_added = _validate_self(containing_node=parent,
defaults=defaults_list)
if is_root_config:
# To ensure config overrides are last, insert the external overrides before the first config override.
insert_idx = len(defaults_list)
for idx, default in enumerate(defaults_list):
if default.is_override():
insert_idx = idx
break
defaults_list[insert_idx:insert_idx] = overrides.append_group_defaults
_update_overrides(defaults_list, overrides, parent, interpolated_subtree)
def add_child(
child_list: List[Union[InputDefault, DefaultsTreeNode]],
new_root_: DefaultsTreeNode,
) -> None:
subtree_ = _create_defaults_tree_impl(
repo=repo,
root=new_root_,
is_root_config=False,
interpolated_subtree=interpolated_subtree,
skip_missing=skip_missing,
overrides=overrides,
)
if subtree_.children is None:
child_list.append(new_root_.node)
else:
child_list.append(subtree_)
for d in reversed(defaults_list):
if d.is_self():
d.update_parent(root.node.parent_base_dir, root.node.get_package())
children.append(d)
else:
if d.is_override():
continue
d.update_parent(parent.get_group_path(),
parent.get_final_package())
if overrides.is_overridden(d):
assert isinstance(d, GroupDefault)
overrides.override_default_option(d)
if isinstance(d, GroupDefault) and d.is_options():
# overriding may change from options to name
if d.is_options():
for item in reversed(d.get_options()):
if "${" in item:
raise ConfigCompositionException(
f"In '{path}': Defaults List interpolation is not supported in options list items"
)
assert d.group is not None
node = ConfigDefault(
path=d.group + "/" + item,
package=d.package,
optional=d.is_optional(),
)
node.update_parent(parent.get_group_path(),
parent.get_final_package())
new_root = DefaultsTreeNode(node=node, parent=root)
add_child(children, new_root)
else:
new_root = DefaultsTreeNode(node=d, parent=root)
add_child(children, new_root)
else:
if d.is_interpolation():
children.append(d)
continue
new_root = DefaultsTreeNode(node=d, parent=root)
add_child(children, new_root)
# processed deferred interpolations
known_choices = _create_interpolation_map(overrides, defaults_list,
self_added)
for idx, dd in enumerate(children):
if isinstance(dd, InputDefault) and dd.is_interpolation():
dd.resolve_interpolation(known_choices)
new_root = DefaultsTreeNode(node=dd, parent=root)
dd.update_parent(parent.get_group_path(),
parent.get_final_package())
subtree = _create_defaults_tree_impl(
repo=repo,
root=new_root,
is_root_config=False,
skip_missing=skip_missing,
interpolated_subtree=True,
overrides=overrides,
)
if subtree.children is not None:
children[idx] = subtree
if len(children) > 0:
root.children = list(reversed(children))
return root
def _load_single_config(
self,
default: DefaultElement,
repo: IConfigRepository,
) -> Tuple[ConfigResult, LoadTrace]:
config_path = default.config_path()
package_override = default.package
ret = repo.load_config(
config_path=config_path,
is_primary_config=default.primary,
package_override=package_override,
)
assert ret is not None
if not isinstance(ret.config, DictConfig):
raise ValueError(
f"Config {config_path} must be a Dictionary, got {type(ret).__name__}"
)
default.search_path = ret.path
schema_provider = None
if not ret.is_schema_source:
schema = None
try:
schema_source = repo.get_schema_source()
schema = schema_source.load_config(
ConfigSource._normalize_file_name(filename=config_path),
is_primary_config=default.primary,
package_override=package_override,
)
except ConfigLoadError:
# schema not found, ignore
pass
if schema is not None:
try:
# if config has a hydra node, remove it during validation and add it back.
# This allows overriding Hydra's configuration without declaring this node
# in every program
hydra = None
hydra_config_group = (
default.config_group is not None
and default.config_group.startswith("hydra/"))
if "hydra" in ret.config and not hydra_config_group:
hydra = ret.config.pop("hydra")
schema_provider = schema.provider
merged = OmegaConf.merge(schema.config, ret.config)
assert isinstance(merged, DictConfig)
if hydra is not None:
with open_dict(merged):
merged.hydra = hydra
ret.config = merged
except OmegaConfBaseException as e:
raise ConfigCompositionException(
f"Error merging '{config_path}' with schema") from e
assert isinstance(merged, DictConfig)
trace = LoadTrace(
config_group=default.config_group,
config_name=default.config_name,
package=default.get_subject_package(),
search_path=ret.path,
parent=default.parent,
provider=ret.provider,
schema_provider=schema_provider,
)
return ret, trace
def _create_defaults_tree_impl(
repo: IConfigRepository,
root: DefaultsTreeNode,
is_root_config: bool,
skip_missing: bool,
interpolated_subtree: bool,
overrides: Overrides,
) -> DefaultsTreeNode:
parent = root.node
children: List[Union[InputDefault, DefaultsTreeNode]] = []
if parent.is_virtual():
if is_root_config:
return _expand_virtual_root(repo, root, overrides, skip_missing)
else:
return root
else:
if is_root_config:
root.node.update_parent("", "")
if is_root_config:
if not repo.config_exists(root.node.get_config_path()):
config_not_found_error(repo=repo, tree=root)
update_package_header(repo=repo, node=parent)
if overrides.is_overridden(parent):
assert isinstance(parent, GroupDefault)
overrides.override_default_option(parent)
# clear package header and obtain updated one from overridden config
# (for the rare case it has changed)
parent.package_header = None
update_package_header(repo=repo, node=parent)
if overrides.is_deleted(parent):
overrides.delete(parent)
return root
overrides.set_known_choice(parent)
if parent.get_name() is None:
return root
if _check_not_missing(repo=repo,
default=parent,
skip_missing=skip_missing):
return root
path = parent.get_config_path()
loaded = repo.load_config(config_path=path)
if loaded is None:
if parent.is_optional():
assert isinstance(parent, GroupDefault)
parent.deleted = True
return root
config_not_found_error(repo=repo, tree=root)
assert loaded is not None
defaults_list = copy.deepcopy(loaded.defaults_list)
if defaults_list is None:
defaults_list = []
self_added = False
if (len(defaults_list) > 0 or is_root_config
and len(overrides.append_group_defaults) > 0):
self_added = _validate_self(containing_node=parent,
defaults=defaults_list)
if is_root_config:
defaults_list.extend(overrides.append_group_defaults)
_update_overrides(defaults_list, overrides, parent,
interpolated_subtree)
for d in reversed(defaults_list):
if d.is_self():
d.update_parent(root.node.parent_base_dir,
root.node.get_package())
children.append(d)
else:
if d.is_override():
continue
new_root = DefaultsTreeNode(node=d, parent=root)
d.update_parent(parent.get_group_path(),
parent.get_final_package())
if d.is_interpolation():
children.append(d)
continue
subtree = _create_defaults_tree_impl(
repo=repo,
root=new_root,
is_root_config=False,
interpolated_subtree=interpolated_subtree,
skip_missing=skip_missing,
overrides=overrides,
)
if subtree.children is None:
children.append(d)
else:
children.append(subtree)
# processed deferred interpolations
known_choices = _create_interpolation_map(overrides, defaults_list,
self_added)
for idx, dd in enumerate(children):
if isinstance(dd, InputDefault) and dd.is_interpolation():
if not parent.primary:
# Interpolations from nested configs would require much more work
# If you have a compelling use case please file an feature request.
path = parent.get_config_path()
raise ConfigCompositionException(
f"In '{path}': Defaults List interpolation is only supported in the primary config"
)
dd.resolve_interpolation(known_choices)
new_root = DefaultsTreeNode(node=dd, parent=root)
dd.update_parent(parent.get_group_path(),
parent.get_final_package())
subtree = _create_defaults_tree_impl(
repo=repo,
root=new_root,
is_root_config=False,
skip_missing=skip_missing,
interpolated_subtree=True,
overrides=overrides,
)
if subtree.children is not None:
children[idx] = subtree
if len(children) > 0:
root.children = list(reversed(children))
return root
def _load_single_config(
self, default: ResultDefault, repo: IConfigRepository
) -> Tuple[ConfigResult, LoadTrace]:
config_path = default.config_path
assert config_path is not None
ret = repo.load_config(config_path=config_path)
assert ret is not None
if not isinstance(ret.config, DictConfig):
raise ValueError(
f"Config {config_path} must be a Dictionary, got {type(ret).__name__}"
)
if not ret.is_schema_source:
schema = None
try:
schema_source = repo.get_schema_source()
cname = ConfigSource._normalize_file_name(filename=config_path)
schema = schema_source.load_config(cname)
except ConfigLoadError:
# schema not found, ignore
pass
if schema is not None:
try:
# TODO: deprecate schema matching in favor of extension via Defaults List
# if primary config has a hydra node, remove it during validation and add it back.
# This allows overriding Hydra's configuration without declaring it's node
# in the schema of every primary config
hydra = None
hydra_config_group = (
default.config_path is not None
and default.config_path.startswith("hydra/")
)
if (
default.primary
and "hydra" in ret.config
and not hydra_config_group
):
hydra = ret.config.pop("hydra")
merged = OmegaConf.merge(schema.config, ret.config)
assert isinstance(merged, DictConfig)
if hydra is not None:
with open_dict(merged):
merged.hydra = hydra
ret.config = merged
except OmegaConfBaseException as e:
raise ConfigCompositionException(
f"Error merging '{config_path}' with schema"
) from e
assert isinstance(merged, DictConfig)
trace = LoadTrace(
config_path=default.config_path,
package=default.package,
parent=default.parent,
is_self=default.is_self,
search_path=ret.path,
provider=ret.provider,
)
ret = self._embed_result_config(ret, default.package)
return ret, trace
