def _get_value_element_as_str(value: ParsedElementType,
space_after_sep: bool = False) -> str:
# str, QuotedString, int, bool, float, List[Any], Dict[str, Any]
comma = ", " if space_after_sep else ","
colon = ": " if space_after_sep else ":"
if value is None:
return "null"
elif isinstance(value, QuotedString):
return value.with_quotes()
elif isinstance(value, list):
s = comma.join([
Override._get_value_element_as_str(
x, space_after_sep=space_after_sep) for x in value
])
return "[" + s + "]"
elif isinstance(value, dict):
s = comma.join([
f"{k}{colon}{Override._get_value_element_as_str(v, space_after_sep=space_after_sep)}"
for k, v in value.items()
])
return "{" + s + "}"
elif isinstance(value, (str, int, bool, float)):
return str(value)
elif is_structured_config(value):
print(value)
return Override._get_value_element_as_str(
OmegaConf.to_container(OmegaConf.structured(value)))
else:
assert False
def _get_value_element_as_str(value: ParsedElementType,
space_after_sep: bool = False) -> str:
# str, QuotedString, int, bool, float, List[Any], Dict[str, Any]
comma = ", " if space_after_sep else ","
colon = ": " if space_after_sep else ":"
if value is None:
return "null"
elif isinstance(value, QuotedString):
return value.with_quotes()
elif isinstance(value, list):
s = comma.join([
Override._get_value_element_as_str(
x, space_after_sep=space_after_sep) for x in value
])
return "[" + s + "]"
elif isinstance(value, dict):
str_items = []
for k, v in value.items():
str_key = Override._get_value_element_as_str(k)
str_value = Override._get_value_element_as_str(
v, space_after_sep=space_after_sep)
str_items.append(f"{str_key}{colon}{str_value}")
return "{" + comma.join(str_items) + "}"
elif isinstance(value, str):
return escape_special_characters(value)
elif isinstance(value, (int, bool, float)):
return str(value)
elif is_structured_config(value):
return Override._get_value_element_as_str(
OmegaConf.to_container(OmegaConf.structured(value)))
else:
assert False
def _is_passthrough(type_: Type[Any]) -> bool:
if type_ is Any or issubclass(type_, (int, float, str, bool, Enum)):
return False
if is_structured_config(type_):
try:
OmegaConf.structured(type_) # verify it's actually legal
except ValidationError as e:
log.debug(
f"Failed to create DictConfig from ({type_.__name__}) : {e}, flagging as passthrough"
)
return True
return False
return True
def is_incompatible(type_: Type[Any]) -> bool:
opt = _resolve_optional(type_)
# Unions are not supported (Except Optional)
if not opt[0] and _is_union(type_):
return True
type_ = opt[1]
if type_ in (type(None), tuple, list, dict):
return False
try:
if is_list_annotation(type_):
lt = get_list_element_type(type_)
return is_incompatible(lt)
if is_dict_annotation(type_):
kvt = get_dict_key_value_types(type_)
if not issubclass(kvt[0], (str, Enum)):
return True
return is_incompatible(kvt[1])
if is_tuple_annotation(type_):
for arg in type_.__args__:
if arg is not ... and is_incompatible(arg):
return True
return False
if get_origin(type_) is Callable:
args = get_args(type_)
for arg in args[0]:
if arg is not ... and is_incompatible(arg):
return True
if is_incompatible(args[1]):
return True
return False
except ValidationError:
return True
if type_ is Any or issubclass(type_, (int, float, str, bool, Enum)):
return False
if is_structured_config(type_):
try:
OmegaConf.structured(type_) # verify it's actually legal
except ValidationError as e:
log.debug(
f"Failed to create DictConfig from ({type_.__name__}) : {e}, flagging as incompatible"
)
return True
return False
return True
def test_merge(inputs: Any, expected: Any) -> None:
configs = [OmegaConf.create(c) for c in inputs]
if isinstance(expected, (dict, list)) or is_structured_config(expected):
merged = OmegaConf.merge(*configs)
assert merged == expected
# test input configs are not changed.
# Note that converting to container without resolving to avoid resolution errors while comparing
for i in range(len(inputs)):
input_i = OmegaConf.create(inputs[i])
orig = OmegaConf.to_container(input_i, resolve=False)
merged2 = OmegaConf.to_container(configs[i], resolve=False)
assert orig == merged2
else:
with expected:
OmegaConf.merge(*configs)
def call(config: Any, *args: Any, **kwargs: Any) -> Any:
"""
:param config: An object describing what to call and what params to use.
Must have a _target_ field.
:param args: optional positional parameters pass-through
:param kwargs: optional named parameters pass-through
:return: the return value from the specified class or method
"""
if OmegaConf.is_none(config):
return None
if isinstance(config, TargetConf) and config._target_ == "???":
# Specific check to give a good warning about failure to annotate _target_ as a string.
raise InstantiationException(
f"Missing value for {type(config).__name__}._target_. Check that it's properly annotated and overridden."
f"\nA common problem is forgetting to annotate _target_ as a string : '_target_: str = ...'"
)
if not (isinstance(config, dict) or OmegaConf.is_config(config)
or is_structured_config(config)):
raise HydraException(
f"Unsupported config type : {type(config).__name__}")
# make a copy to ensure we do not change the provided object
config_copy = OmegaConf.structured(config)
if OmegaConf.is_config(config):
config_copy._set_parent(config._get_parent())
config = config_copy
cls = "<unknown>"
try:
assert isinstance(config, DictConfig)
OmegaConf.set_readonly(config, False)
OmegaConf.set_struct(config, False)
cls = _get_cls_name(config)
type_or_callable = _locate(cls)
if isinstance(type_or_callable, type):
return _instantiate_class(type_or_callable, config, *args,
**kwargs)
else:
assert callable(type_or_callable)
return _call_callable(type_or_callable, config, *args, **kwargs)
except InstantiationException as e:
raise e
except Exception as e:
raise HydraException(f"Error calling '{cls}' : {e}") from e
def check_node_metadata(
node: Container,
type_hint: Any,
key_type: Any,
elt_type: Any,
obj_type: Any,
) -> None:
value_optional, value_ref_type = _resolve_optional(type_hint)
assert node._metadata.optional == value_optional
assert node._metadata.ref_type == value_ref_type
assert node._metadata.key_type == key_type
assert node._metadata.element_type == elt_type
assert node._metadata.object_type == obj_type
if is_dict_annotation(value_ref_type) or is_structured_config(
value_ref_type):
assert isinstance(node, DictConfig)
elif is_list_annotation(value_ref_type):
assert isinstance(node, ListConfig)
def test_merge(
inputs: Any,
expected: Any,
merge_function: Any,
input_unchanged: bool,
) -> None:
configs = [OmegaConf.create(c) for c in inputs]
if isinstance(
expected,
(MutableMapping, MutableSequence)) or is_structured_config(expected):
merged = merge_function(*configs)
assert merged == expected
if input_unchanged:
# test input configs are not changed.
# Note that converting to container without resolving to avoid resolution errors while comparing
for i in range(len(inputs)):
input_i = OmegaConf.create(inputs[i])
orig = OmegaConf.to_container(input_i, resolve=False)
merged2 = OmegaConf.to_container(configs[i], resolve=False)
assert orig == merged2
else:
with expected:
merge_function(*configs)
def instantiate(config: Any, *args: Any, **kwargs: Any) -> Any:
"""
:param config: An config object describing what to call and what params to use.
In addition to the parameters, the config must contain:
_target_ : target class or callable name (str)
And may contain:
_args_: List-like of positional arguments to pass to the target
_recursive_: Construct nested objects as well (bool).
True by default.
may be overridden via a _recursive_ key in
the kwargs
_convert_: Conversion strategy
none : Passed objects are DictConfig and ListConfig, default
partial : Passed objects are converted to dict and list, with
the exception of Structured Configs (and their fields).
all : Passed objects are dicts, lists and primitives without
a trace of OmegaConf containers
_args_: List-like of positional arguments
:param args: Optional positional parameters pass-through
:param kwargs: Optional named parameters to override
parameters in the config object. Parameters not present
in the config objects are being passed as is to the target.
IMPORTANT: dataclasses instances in kwargs are interpreted as config
and cannot be used as passthrough
:return: if _target_ is a class name: the instantiated object
if _target_ is a callable: the return value of the call
"""
# Return None if config is None
if config is None or OmegaConf.is_none(config):
return None
# TargetConf edge case
if isinstance(config, TargetConf) and config._target_ == "???":
# Specific check to give a good warning about failure to annotate _target_ as a string.
raise InstantiationException(
f"Missing value for {type(config).__name__}._target_. Check that it's properly annotated and overridden."
f"\nA common problem is forgetting to annotate _target_ as a string : '_target_: str = ...'"
)
if isinstance(config, dict):
config = _prepare_input_dict(config)
kwargs = _prepare_input_dict(kwargs)
# Structured Config always converted first to OmegaConf
if is_structured_config(config) or isinstance(config, dict):
config = OmegaConf.structured(config, flags={"allow_objects": True})
if OmegaConf.is_dict(config):
# Finalize config (convert targets to strings, merge with kwargs)
config_copy = copy.deepcopy(config)
config_copy._set_flag(flags=["allow_objects", "struct", "readonly"],
values=[True, False, False])
config_copy._set_parent(config._get_parent())
config = config_copy
if kwargs:
config = OmegaConf.merge(config, kwargs)
_recursive_ = config.pop(_Keys.RECURSIVE, True)
_convert_ = config.pop(_Keys.CONVERT, ConvertMode.NONE)
return instantiate_node(config,
*args,
recursive=_recursive_,
convert=_convert_)
else:
raise InstantiationException(
"Top level config has to be OmegaConf DictConfig, plain dict, or a Structured Config class or instance"
)
def instantiate(config: Any, *args: Any, **kwargs: Any) -> Any:
"""
:param config: An config object describing what to call and what params to use.
In addition to the parameters, the config must contain:
_target_ : target class or callable name (str)
And may contain:
_recursive_: Construct nested objects as well (bool).
True by default.
may be overridden via a _recursive_ key in
the kwargs
_convert_: Conversion strategy
none : Passed objects are DictConfig and ListConfig, default
partial : Passed objects are converted to dict and list, with
the exception of Structured Configs (and their fields).
all : Passed objects are dicts, lists and primitives without
a trace of OmegaConf containers
:param args: Optional positional parameters pass-through
:param kwargs: Optional named parameters to override
parameters in the config object. Parameters not present
in the config objects are being passed as is to the target.
:return: if _target_ is a class name: the instantiated object
if _target_ is a callable: the return value of the call
"""
if OmegaConf.is_none(config):
return None
if isinstance(config, TargetConf) and config._target_ == "???":
# Specific check to give a good warning about failure to annotate _target_ as a string.
raise InstantiationException(
f"Missing value for {type(config).__name__}._target_. Check that it's properly annotated and overridden."
f"\nA common problem is forgetting to annotate _target_ as a string : '_target_: str = ...'"
)
if not (
isinstance(config, dict)
or OmegaConf.is_config(config)
or is_structured_config(config)
):
raise HydraException(f"Unsupported config type : {type(config).__name__}")
if isinstance(config, dict):
configc = config.copy()
_convert_container_targets_to_strings(configc)
config = configc
kwargsc = kwargs.copy()
_convert_container_targets_to_strings(kwargsc)
kwargs = kwargsc
# make a copy to ensure we do not change the provided object
config_copy = OmegaConf.structured(config, flags={"allow_objects": True})
if OmegaConf.is_config(config):
config_copy._set_parent(config._get_parent())
config = config_copy
assert OmegaConf.is_config(config)
OmegaConf.set_readonly(config, False)
OmegaConf.set_struct(config, False)
target = _get_target_type(config, kwargs)
try:
config._set_flag("allow_objects", True)
final_kwargs = _get_kwargs(config, **kwargs)
convert = _pop_convert_mode(final_kwargs)
if convert == ConvertMode.PARTIAL:
final_kwargs = OmegaConf.to_container(
final_kwargs, resolve=True, exclude_structured_configs=True
)
return target(*args, **final_kwargs)
elif convert == ConvertMode.ALL:
final_kwargs = OmegaConf.to_container(final_kwargs, resolve=True)
return target(*args, **final_kwargs)
elif convert == ConvertMode.NONE:
return target(*args, **final_kwargs)
else:
assert False
except Exception as e:
# preserve the original exception backtrace
raise type(e)(
f"Error instantiating/calling '{_convert_target_to_string(target)}' : {e}"
).with_traceback(sys.exc_info()[2])
def instantiate(config: Any, *args: Any, **kwargs: Any) -> Any:
"""
:param config: An config object describing what to call and what params to use.
In addition to the parameters, the config must contain:
_target_ : target class or callable name (str)
And may contain:
_args_: List-like of positional arguments to pass to the target
_recursive_: Construct nested objects as well (bool).
True by default.
may be overridden via a _recursive_ key in
the kwargs
_convert_: Conversion strategy
none : Passed objects are DictConfig and ListConfig, default
partial : Passed objects are converted to dict and list, with
the exception of Structured Configs (and their fields).
all : Passed objects are dicts, lists and primitives without
a trace of OmegaConf containers
_partial_: If True, return functools.partial wrapped method or object
False by default. Configure per target.
:param args: Optional positional parameters pass-through
:param kwargs: Optional named parameters to override
parameters in the config object. Parameters not present
in the config objects are being passed as is to the target.
IMPORTANT: dataclasses instances in kwargs are interpreted as config
and cannot be used as passthrough
:return: if _target_ is a class name: the instantiated object
if _target_ is a callable: the return value of the call
"""
# Return None if config is None
if config is None:
return None
# TargetConf edge case
if isinstance(config, TargetConf) and config._target_ == "???":
# Specific check to give a good warning about failure to annotate _target_ as a string.
raise InstantiationException(
dedent(f"""\
Config has missing value for key `_target_`, cannot instantiate.
Config type: {type(config).__name__}
Check that the `_target_` key in your dataclass is properly annotated and overridden.
A common problem is forgetting to annotate _target_ as a string : '_target_: str = ...'"""
))
# TODO: print full key
if isinstance(config, (dict, list)):
config = _prepare_input_dict_or_list(config)
kwargs = _prepare_input_dict_or_list(kwargs)
# Structured Config always converted first to OmegaConf
if is_structured_config(config) or isinstance(config, (dict, list)):
config = OmegaConf.structured(config, flags={"allow_objects": True})
if OmegaConf.is_dict(config):
# Finalize config (convert targets to strings, merge with kwargs)
config_copy = copy.deepcopy(config)
config_copy._set_flag(flags=["allow_objects", "struct", "readonly"],
values=[True, False, False])
config_copy._set_parent(config._get_parent())
config = config_copy
if kwargs:
config = OmegaConf.merge(config, kwargs)
OmegaConf.resolve(config)
_recursive_ = config.pop(_Keys.RECURSIVE, True)
_convert_ = config.pop(_Keys.CONVERT, ConvertMode.NONE)
_partial_ = config.pop(_Keys.PARTIAL, False)
return instantiate_node(config,
*args,
recursive=_recursive_,
convert=_convert_,
partial=_partial_)
elif OmegaConf.is_list(config):
# Finalize config (convert targets to strings, merge with kwargs)
config_copy = copy.deepcopy(config)
config_copy._set_flag(flags=["allow_objects", "struct", "readonly"],
values=[True, False, False])
config_copy._set_parent(config._get_parent())
config = config_copy
OmegaConf.resolve(config)
_recursive_ = kwargs.pop(_Keys.RECURSIVE, True)
_convert_ = kwargs.pop(_Keys.CONVERT, ConvertMode.NONE)
_partial_ = kwargs.pop(_Keys.PARTIAL, False)
if _partial_:
raise InstantiationException(
"The _partial_ keyword is not compatible with top-level list instantiation"
)
return instantiate_node(config,
*args,
recursive=_recursive_,
convert=_convert_,
partial=_partial_)
else:
raise InstantiationException(
dedent(f"""\
Cannot instantiate config of type {type(config).__name__}.
Top level config must be an OmegaConf DictConfig/ListConfig object,
a plain dict/list, or a Structured Config class or instance."""
))
def instantiate(config: Any, *args: Any, **kwargs: Any) -> Any:
"""
:param config: An config object describing what to call and what params to use.
In addition to the parameters, the config must contain:
_target_ : target class or callable name (str)
_recursive_: Construct nested objects as well (bool).
True by default.
may be overridden via a _recursive_ key in
the kwargs
:param args: Optional positional parameters pass-through
:param kwargs: Optional named parameters to override
parameters in the config object. Parameters not present
in the config objects are being passed as is to the target.
:return: if _target_ is a class name: the instantiated object
if _target_ is a callable: the return value of the call
"""
if OmegaConf.is_none(config):
return None
if isinstance(config, TargetConf) and config._target_ == "???":
# Specific check to give a good warning about failure to annotate _target_ as a string.
raise InstantiationException(
f"Missing value for {type(config).__name__}._target_. Check that it's properly annotated and overridden."
f"\nA common problem is forgetting to annotate _target_ as a string : '_target_: str = ...'"
)
if not (
isinstance(config, dict)
or OmegaConf.is_config(config)
or is_structured_config(config)
):
raise HydraException(f"Unsupported config type : {type(config).__name__}")
if isinstance(config, dict):
configc = config.copy()
_convert_container_targets_to_strings(configc)
config = configc
kwargsc = kwargs.copy()
_convert_container_targets_to_strings(kwargsc)
kwargs = kwargsc
# make a copy to ensure we do not change the provided object
config_copy = OmegaConf.structured(config, flags={"allow_objects": True})
if OmegaConf.is_config(config):
config_copy._set_parent(config._get_parent())
config = config_copy
assert OmegaConf.is_config(config)
OmegaConf.set_readonly(config, False)
OmegaConf.set_struct(config, False)
target = _get_target_type(config, kwargs)
try:
config._set_flag("allow_objects", True)
final_kwargs = _get_kwargs(config, **kwargs)
return target(*args, **final_kwargs)
except Exception as e:
raise type(e)(
f"Error instantiating/calling '{_convert_target_to_string(target)}' : {e}"
)
