def resolve_to_config_type(dagster_type):
from .field_utils import convert_fields_to_dict_type
# Short circuit if it's already a Config Type
if isinstance(dagster_type, ConfigType):
return dagster_type
if isinstance(dagster_type, dict):
return convert_fields_to_dict_type(dagster_type)
if isinstance(dagster_type, list):
if len(dagster_type) != 1:
raise DagsterInvalidDefinitionError(
'Array specifications must only be of length 1')
inner_type = resolve_to_config_type(dagster_type[0])
if not inner_type:
raise DagsterInvalidDefinitionError(
'Invalid member of array specification: {value} in list {the_list}'
.format(value=repr(dagster_type[0]), the_list=dagster_type))
return Array(inner_type)
from dagster.core.types.dagster_type import DagsterType, List, ListType
from dagster.core.types.python_set import Set, _TypedPythonSet
from dagster.core.types.python_tuple import Tuple, _TypedPythonTuple
if _is_config_type_class(dagster_type):
check.param_invariant(
False,
'dagster_type',
'Cannot pass a config type class to resolve_to_config_type. Got {dagster_type}'
.format(dagster_type=dagster_type),
)
if isinstance(dagster_type, type) and issubclass(dagster_type,
DagsterType):
raise DagsterInvalidDefinitionError(
'You have passed a DagsterType class {dagster_type} to the config system. '
'The DagsterType and config schema systems are separate. '
'Valid config values are:\n{desc}'.format(
dagster_type=repr(dagster_type),
desc=VALID_CONFIG_DESC,
))
if is_typing_type(dagster_type):
raise DagsterInvalidDefinitionError((
'You have passed in {dagster_type} to the config system. Types from '
'the typing module in python are not allowed in the config system. '
'You must use types that are imported from dagster or primitive types '
'such as bool, int, etc.').format(dagster_type=dagster_type))
if dagster_type is List or isinstance(dagster_type, ListType):
raise DagsterInvalidDefinitionError(
'Cannot use List in the context of config. ' +
helpful_list_error_string())
if dagster_type is Set or isinstance(dagster_type, _TypedPythonSet):
raise DagsterInvalidDefinitionError(
'Cannot use Set in the context of a config field. ' +
helpful_list_error_string())
if dagster_type is Tuple or isinstance(dagster_type, _TypedPythonTuple):
raise DagsterInvalidDefinitionError(
'Cannot use Tuple in the context of a config field. ' +
helpful_list_error_string())
if isinstance(dagster_type, DagsterType):
raise DagsterInvalidDefinitionError((
'You have passed an instance of DagsterType {type_name} to the config '
'system (Repr of type: {dagster_type}). '
'The DagsterType and config schema systems are separate. '
'Valid config values are:\n{desc}').format(
type_name=dagster_type.name
if dagster_type.name else dagster_type.key,
dagster_type=repr(dagster_type),
desc=VALID_CONFIG_DESC,
), )
# If we are passed here either:
# 1) We have been passed a python builtin
# 2) We have been a dagster wrapping type that needs to be convert its config variant
# e.g. dagster.List
# 2) We have been passed an invalid thing. We return False to signify this. It is
# up to callers to report a reasonable error.
from dagster.primitive_mapping import (
remap_python_builtin_for_config,
is_supported_config_python_builtin,
)
if is_supported_config_python_builtin(dagster_type):
return remap_python_builtin_for_config(dagster_type)
if dagster_type is None:
return ConfigAnyInstance
if BuiltinEnum.contains(dagster_type):
return ConfigType.from_builtin_enum(dagster_type)
# This means that this is an error and we are return False to a callsite
# We do the error reporting there because those callsites have more context
return False
def resolve_dagster_type(dagster_type: object) -> DagsterType:
# circular dep
from .python_dict import PythonDict, Dict
from .python_set import PythonSet, DagsterSetApi
from .python_tuple import PythonTuple, DagsterTupleApi
from .transform_typing import transform_typing_type
from dagster.primitive_mapping import (
remap_python_builtin_for_runtime,
is_supported_runtime_python_builtin,
)
from dagster.utils.typing_api import is_typing_type
check.invariant(
not (isinstance(dagster_type, type) and issubclass(dagster_type, ConfigType)),
"Cannot resolve a config type to a runtime type",
)
check.invariant(
not (isinstance(dagster_type, type) and issubclass(dagster_type, DagsterType)),
"Do not pass runtime type classes. Got {}".format(dagster_type),
)
# First check to see if it is part of python's typing library
if is_typing_type(dagster_type):
dagster_type = transform_typing_type(dagster_type)
if isinstance(dagster_type, DagsterType):
return dagster_type
# Test for unhashable objects -- this is if, for instance, someone has passed us an instance of
# a dict where they meant to pass dict or Dict, etc.
try:
hash(dagster_type)
except TypeError:
raise DagsterInvalidDefinitionError(
DAGSTER_INVALID_TYPE_ERROR_MESSAGE.format(
additional_msg=(
", which isn't hashable. Did you pass an instance of a type instead of "
"the type?"
),
dagster_type=str(dagster_type),
)
)
if BuiltinEnum.contains(dagster_type):
return DagsterType.from_builtin_enum(dagster_type)
if is_supported_runtime_python_builtin(dagster_type):
return remap_python_builtin_for_runtime(dagster_type)
if dagster_type is None:
return Any
if dagster_type is Dict:
return PythonDict
if isinstance(dagster_type, DagsterTupleApi):
return PythonTuple
if isinstance(dagster_type, DagsterSetApi):
return PythonSet
if isinstance(dagster_type, DagsterListApi):
return List(Any)
if isinstance(dagster_type, type):
return resolve_python_type_to_dagster_type(dagster_type)
raise DagsterInvalidDefinitionError(
DAGSTER_INVALID_TYPE_ERROR_MESSAGE.format(
dagster_type=str(dagster_type), additional_msg="."
)
)
def resolve_to_config_type(dagster_type):
from dagster.core.types.wrapping.mapping import (
remap_python_builtin_for_config,
is_supported_config_python_builtin,
)
from dagster.core.types.runtime.runtime_type import RuntimeType
if _is_config_type_class(dagster_type):
check.param_invariant(
False,
'dagster_type',
'Cannot pass a config type class to resolve_to_config_type. Got {dagster_type}'
.format(dagster_type=dagster_type),
)
check.invariant(
not (isinstance(dagster_type, type)
and issubclass(dagster_type, RuntimeType)),
'Cannot resolve a runtime type to a config type',
)
if is_typing_type(dagster_type):
raise DagsterInvariantViolationError((
'You have passed in {dagster_type} in the config system. Types from '
'the typing module in python are not allowed in the config system. '
'You must use types that are imported from dagster or primitive types '
'such as bool, int, etc.').format(dagster_type=dagster_type))
if isinstance(dagster_type, (WrappingSetType, DagsterSetApi)):
raise DagsterInvalidDefinitionError(
'Cannot use Set in the context of a config field. Please use List instead.'
)
if isinstance(dagster_type, (WrappingTupleType, DagsterTupleApi)):
raise DagsterInvalidDefinitionError(
'Cannot use Tuple in the context of a config field. Please use List instead.'
)
# Short circuit if it's already a Config Type
if isinstance(dagster_type, ConfigType):
return dagster_type
# If we are passed here either:
# 1) We have been passed a python builtin
# 2) We have been a dagster wrapping type that needs to be convert its config varient
# e.g. dagster.List
# 2) We have been passed an invalid thing. We return False to signify this. It is
# up to callers to report a reasonable error.
if is_supported_config_python_builtin(dagster_type):
return remap_python_builtin_for_config(dagster_type)
if dagster_type is None:
return ConfigAnyInstance
if BuiltinEnum.contains(dagster_type):
return ConfigType.from_builtin_enum(dagster_type)
if isinstance(dagster_type, (WrappingListType, DagsterListApi)):
return resolve_to_config_list(dagster_type)
if isinstance(dagster_type, WrappingNullableType):
return resolve_to_config_nullable(dagster_type)
# This means that this is an error and we are return False to a callsite
# We do the error reporting there because those callsites have more context
return False
def test_is_typing_type():
assert is_typing_type("foobar") is False
assert is_typing_type(1) is False
assert is_typing_type(dict) is False
assert is_typing_type(int) is False
assert is_typing_type(list) is False
assert is_typing_type(None) is False
assert is_typing_type(set) is False
assert is_typing_type(tuple) is False
assert is_typing_type(typing.Dict) is True
assert is_typing_type(typing.Dict[int, str]) is True
assert is_typing_type(typing.Dict[str, typing.Dict[str, typing.Dict]]) is True
assert is_typing_type(typing.Dict[str, typing.Dict]) is True
assert is_typing_type(typing.Dict[str, typing.List]) is True
assert is_typing_type(typing.Dict[str, typing.Optional[typing.Dict]]) is True
assert is_typing_type(typing.Dict[str, typing.Tuple]) is True
assert is_typing_type(typing.List) is True
assert is_typing_type(typing.List[int]) is True
assert is_typing_type(typing.Optional) is False
assert is_typing_type(typing.Optional[int]) is True
assert is_typing_type(typing.Set) is True
assert is_typing_type(typing.Set[int]) is True
assert is_typing_type(typing.Set[typing.Dict[str, typing.Dict]]) is True
assert is_typing_type(typing.Set[typing.Dict]) is True
assert is_typing_type(typing.Set[typing.List]) is True
assert is_typing_type(typing.Set[typing.Optional[typing.Dict]]) is True
assert is_typing_type(typing.Set[typing.Tuple]) is True
assert is_typing_type(typing.Tuple) is True
assert is_typing_type(typing.Tuple[int, str]) is True
assert is_typing_type(typing.Tuple[str, typing.Dict[str, typing.Dict]]) is True
assert is_typing_type(typing.Tuple[str, typing.Dict]) is True
assert is_typing_type(typing.Tuple[str, typing.List]) is True
assert is_typing_type(typing.Tuple[str, typing.Optional[typing.Dict]]) is True
assert is_typing_type(typing.Tuple[str, typing.Tuple]) is True
def resolve_to_runtime_type(dagster_type):
# circular dep
from .python_dict import PythonDict
from .python_set import PythonSet
from .python_tuple import PythonTuple
from dagster.core.types.config.config_type import ConfigType
from dagster.core.types.wrapping.mapping import (
remap_python_builtin_for_runtime,
is_supported_runtime_python_builtin,
)
from dagster.core.types.wrapping.wrapping import transform_typing_type
from dagster.utils.typing_api import is_typing_type
check.invariant(
not (isinstance(dagster_type, type)
and issubclass(dagster_type, ConfigType)),
'Cannot resolve a config type to a runtime type',
)
check.invariant(
not (isinstance(dagster_type, type)
and issubclass(dagster_type, RuntimeType)),
'Do not pass runtime type classes. Got {}'.format(dagster_type),
)
# First check to see if it part of python's typing library
# Transform to our wrapping type system.
if is_typing_type(dagster_type):
dagster_type = transform_typing_type(dagster_type)
if isinstance(dagster_type, RuntimeType):
return dagster_type
# Test for unhashable objects -- this is if, for instance, someone has passed us an instance of
# a dict where they meant to pass dict or Dict, etc.
try:
hash(dagster_type)
except TypeError:
raise DagsterInvalidDefinitionError(
DAGSTER_INVALID_TYPE_ERROR_MESSAGE.format(
additional_msg=
(', which isn\'t hashable. Did you pass an instance of a type instead of '
'the type?'),
dagster_type=str(dagster_type),
))
if is_supported_runtime_python_builtin(dagster_type):
return remap_python_builtin_for_runtime(dagster_type)
if dagster_type is None:
return Any
if dagster_type in __RUNTIME_TYPE_REGISTRY:
return __RUNTIME_TYPE_REGISTRY[dagster_type]
if dagster_type is Dict:
return PythonDict
if isinstance(dagster_type, DagsterTupleApi):
return PythonTuple
if isinstance(dagster_type, DagsterSetApi):
return PythonSet
if isinstance(dagster_type, DagsterListApi):
return resolve_to_runtime_list(WrappingListType(BuiltinEnum.ANY))
if BuiltinEnum.contains(dagster_type):
return RuntimeType.from_builtin_enum(dagster_type)
if isinstance(dagster_type, WrappingListType):
return resolve_to_runtime_list(dagster_type)
if isinstance(dagster_type, WrappingSetType):
return resolve_to_runtime_set(dagster_type)
if isinstance(dagster_type, WrappingTupleType):
return resolve_to_runtime_tuple(dagster_type)
if isinstance(dagster_type, WrappingNullableType):
return resolve_to_runtime_nullable(dagster_type)
if not isinstance(dagster_type, type):
raise DagsterInvalidDefinitionError(
DAGSTER_INVALID_TYPE_ERROR_MESSAGE.format(
dagster_type=str(dagster_type), additional_msg='.'))
check.inst(dagster_type, type)
if dagster_type in __ANONYMOUS_TYPE_REGISTRY:
return __ANONYMOUS_TYPE_REGISTRY[dagster_type]
return create_anonymous_type(dagster_type)
def resolve_to_config_type(dagster_type) -> Union[ConfigType, bool]:
from .field_utils import convert_fields_to_dict_type
# Short circuit if it's already a Config Type
if isinstance(dagster_type, ConfigType):
return dagster_type
if isinstance(dagster_type, dict):
return convert_fields_to_dict_type(dagster_type)
if isinstance(dagster_type, list):
if len(dagster_type) != 1:
raise DagsterInvalidDefinitionError(
"Array specifications must only be of length 1")
inner_type = resolve_to_config_type(dagster_type[0])
if not inner_type:
raise DagsterInvalidDefinitionError(
"Invalid member of array specification: {value} in list {the_list}"
.format(value=repr(dagster_type[0]), the_list=dagster_type))
return Array(inner_type)
from dagster.core.types.dagster_type import DagsterType, List, ListType
from dagster.core.types.python_set import Set, _TypedPythonSet
from dagster.core.types.python_tuple import Tuple, _TypedPythonTuple
if _is_config_type_class(dagster_type):
check.param_invariant(
False,
"dagster_type",
f"Cannot pass config type class {dagster_type} to resolve_to_config_type. "
"This error usually occurs when you pass a dagster config type class instead of a class instance into "
'another dagster config type. E.g. "Noneable(Permissive)" should instead be "Noneable(Permissive())".',
)
if isinstance(dagster_type, type) and issubclass(dagster_type,
DagsterType):
raise DagsterInvalidDefinitionError(
"You have passed a DagsterType class {dagster_type} to the config system. "
"The DagsterType and config schema systems are separate. "
"Valid config values are:\n{desc}".format(
dagster_type=repr(dagster_type),
desc=VALID_CONFIG_DESC,
))
if is_closed_python_optional_type(dagster_type):
raise DagsterInvalidDefinitionError(
"Cannot use typing.Optional as a config type. If you want this field to be "
"optional, please use Field(<type>, is_required=False), and if you want this field to "
"be required, but accept a value of None, use dagster.Noneable(<type>)."
)
if is_typing_type(dagster_type):
raise DagsterInvalidDefinitionError((
"You have passed in {dagster_type} to the config system. Types from "
"the typing module in python are not allowed in the config system. "
"You must use types that are imported from dagster or primitive types "
"such as bool, int, etc.").format(dagster_type=dagster_type))
if dagster_type is List or isinstance(dagster_type, ListType):
raise DagsterInvalidDefinitionError(
"Cannot use List in the context of config. " +
helpful_list_error_string())
if dagster_type is Set or isinstance(dagster_type, _TypedPythonSet):
raise DagsterInvalidDefinitionError(
"Cannot use Set in the context of a config field. " +
helpful_list_error_string())
if dagster_type is Tuple or isinstance(dagster_type, _TypedPythonTuple):
raise DagsterInvalidDefinitionError(
"Cannot use Tuple in the context of a config field. " +
helpful_list_error_string())
if isinstance(dagster_type, DagsterType):
raise DagsterInvalidDefinitionError((
"You have passed an instance of DagsterType {type_name} to the config "
"system (Repr of type: {dagster_type}). "
"The DagsterType and config schema systems are separate. "
"Valid config values are:\n{desc}").format(
type_name=dagster_type.display_name,
dagster_type=repr(dagster_type),
desc=VALID_CONFIG_DESC,
), )
# If we are passed here either:
# 1) We have been passed a python builtin
# 2) We have been a dagster wrapping type that needs to be convert its config variant
# e.g. dagster.List
# 2) We have been passed an invalid thing. We return False to signify this. It is
# up to callers to report a reasonable error.
from dagster.primitive_mapping import (
remap_python_builtin_for_config,
is_supported_config_python_builtin,
)
if BuiltinEnum.contains(dagster_type):
return ConfigType.from_builtin_enum(dagster_type)
if is_supported_config_python_builtin(dagster_type):
return remap_python_builtin_for_config(dagster_type)
if dagster_type is None:
return ConfigAnyInstance
# This means that this is an error and we are return False to a callsite
# We do the error reporting there because those callsites have more context
return False
