def resolve_dagster_type(dagster_type: object) -> DagsterType: # circular dep from .python_dict import PythonDict, Dict from .python_set import PythonSet, DagsterSetApi from .python_tuple import PythonTuple, DagsterTupleApi from .transform_typing import transform_typing_type from dagster.primitive_mapping import ( remap_python_builtin_for_runtime, is_supported_runtime_python_builtin, ) from dagster.utils.typing_api import is_typing_type check.invariant( not (isinstance(dagster_type, type) and issubclass(dagster_type, ConfigType)), "Cannot resolve a config type to a runtime type", ) check.invariant( not (isinstance(dagster_type, type) and issubclass(dagster_type, DagsterType)), "Do not pass runtime type classes. Got {}".format(dagster_type), ) # First check to see if it is part of python's typing library if is_typing_type(dagster_type): dagster_type = transform_typing_type(dagster_type) if isinstance(dagster_type, DagsterType): return dagster_type # Test for unhashable objects -- this is if, for instance, someone has passed us an instance of # a dict where they meant to pass dict or Dict, etc. try: hash(dagster_type) except TypeError: raise DagsterInvalidDefinitionError( DAGSTER_INVALID_TYPE_ERROR_MESSAGE.format( additional_msg=( ", which isn't hashable. Did you pass an instance of a type instead of " "the type?" ), dagster_type=str(dagster_type), ) ) if BuiltinEnum.contains(dagster_type): return DagsterType.from_builtin_enum(dagster_type) if is_supported_runtime_python_builtin(dagster_type): return remap_python_builtin_for_runtime(dagster_type) if dagster_type is None: return Any if dagster_type is Dict: return PythonDict if isinstance(dagster_type, DagsterTupleApi): return PythonTuple if isinstance(dagster_type, DagsterSetApi): return PythonSet if isinstance(dagster_type, DagsterListApi): return List(Any) if isinstance(dagster_type, type): return resolve_python_type_to_dagster_type(dagster_type) raise DagsterInvalidDefinitionError( DAGSTER_INVALID_TYPE_ERROR_MESSAGE.format( dagster_type=str(dagster_type), additional_msg="." ) )
def resolve_to_config_type(dagster_type): from .field_utils import convert_fields_to_dict_type # Short circuit if it's already a Config Type if isinstance(dagster_type, ConfigType): return dagster_type if isinstance(dagster_type, dict): return convert_fields_to_dict_type(dagster_type) if isinstance(dagster_type, list): if len(dagster_type) != 1: raise DagsterInvalidDefinitionError( 'Array specifications must only be of length 1') inner_type = resolve_to_config_type(dagster_type[0]) if not inner_type: raise DagsterInvalidDefinitionError( 'Invalid member of array specification: {value} in list {the_list}' .format(value=repr(dagster_type[0]), the_list=dagster_type)) return Array(inner_type) from dagster.core.types.dagster_type import DagsterType, List, ListType from dagster.core.types.python_set import Set, _TypedPythonSet from dagster.core.types.python_tuple import Tuple, _TypedPythonTuple if _is_config_type_class(dagster_type): check.param_invariant( False, 'dagster_type', 'Cannot pass a config type class to resolve_to_config_type. Got {dagster_type}' .format(dagster_type=dagster_type), ) if isinstance(dagster_type, type) and issubclass(dagster_type, DagsterType): raise DagsterInvalidDefinitionError( 'You have passed a DagsterType class {dagster_type} to the config system. ' 'The DagsterType and config schema systems are separate. ' 'Valid config values are:\n{desc}'.format( dagster_type=repr(dagster_type), desc=VALID_CONFIG_DESC, )) if is_typing_type(dagster_type): raise DagsterInvalidDefinitionError(( 'You have passed in {dagster_type} to the config system. Types from ' 'the typing module in python are not allowed in the config system. ' 'You must use types that are imported from dagster or primitive types ' 'such as bool, int, etc.').format(dagster_type=dagster_type)) if dagster_type is List or isinstance(dagster_type, ListType): raise DagsterInvalidDefinitionError( 'Cannot use List in the context of config. ' + helpful_list_error_string()) if dagster_type is Set or isinstance(dagster_type, _TypedPythonSet): raise DagsterInvalidDefinitionError( 'Cannot use Set in the context of a config field. ' + helpful_list_error_string()) if dagster_type is Tuple or isinstance(dagster_type, _TypedPythonTuple): raise DagsterInvalidDefinitionError( 'Cannot use Tuple in the context of a config field. ' + helpful_list_error_string()) if isinstance(dagster_type, DagsterType): raise DagsterInvalidDefinitionError(( 'You have passed an instance of DagsterType {type_name} to the config ' 'system (Repr of type: {dagster_type}). ' 'The DagsterType and config schema systems are separate. ' 'Valid config values are:\n{desc}').format( type_name=dagster_type.name if dagster_type.name else dagster_type.key, dagster_type=repr(dagster_type), desc=VALID_CONFIG_DESC, ), ) # If we are passed here either: # 1) We have been passed a python builtin # 2) We have been a dagster wrapping type that needs to be convert its config variant # e.g. dagster.List # 2) We have been passed an invalid thing. We return False to signify this. It is # up to callers to report a reasonable error. from dagster.primitive_mapping import ( remap_python_builtin_for_config, is_supported_config_python_builtin, ) if is_supported_config_python_builtin(dagster_type): return remap_python_builtin_for_config(dagster_type) if dagster_type is None: return ConfigAnyInstance if BuiltinEnum.contains(dagster_type): return ConfigType.from_builtin_enum(dagster_type) # This means that this is an error and we are return False to a callsite # We do the error reporting there because those callsites have more context return False
def from_builtin_enum(builtin_enum): check.invariant(BuiltinEnum.contains(builtin_enum), "param must be member of BuiltinEnum") return _CONFIG_MAP[builtin_enum]
def from_builtin_enum(builtin_enum) -> "DagsterType": check.invariant(BuiltinEnum.contains(builtin_enum), "must be member of BuiltinEnum") return _RUNTIME_MAP[builtin_enum]
def from_builtin_enum(builtin_enum): check.invariant(BuiltinEnum.contains(builtin_enum), 'must be member of BuiltinEnum') return _RUNTIME_MAP[builtin_enum]
def resolve_to_config_type(dagster_type) -> Union[ConfigType, bool]: from .field_utils import convert_fields_to_dict_type # Short circuit if it's already a Config Type if isinstance(dagster_type, ConfigType): return dagster_type if isinstance(dagster_type, dict): return convert_fields_to_dict_type(dagster_type) if isinstance(dagster_type, list): if len(dagster_type) != 1: raise DagsterInvalidDefinitionError( "Array specifications must only be of length 1") inner_type = resolve_to_config_type(dagster_type[0]) if not inner_type: raise DagsterInvalidDefinitionError( "Invalid member of array specification: {value} in list {the_list}" .format(value=repr(dagster_type[0]), the_list=dagster_type)) return Array(inner_type) from dagster.core.types.dagster_type import DagsterType, List, ListType from dagster.core.types.python_set import Set, _TypedPythonSet from dagster.core.types.python_tuple import Tuple, _TypedPythonTuple if _is_config_type_class(dagster_type): check.param_invariant( False, "dagster_type", f"Cannot pass config type class {dagster_type} to resolve_to_config_type. " "This error usually occurs when you pass a dagster config type class instead of a class instance into " 'another dagster config type. E.g. "Noneable(Permissive)" should instead be "Noneable(Permissive())".', ) if isinstance(dagster_type, type) and issubclass(dagster_type, DagsterType): raise DagsterInvalidDefinitionError( "You have passed a DagsterType class {dagster_type} to the config system. " "The DagsterType and config schema systems are separate. " "Valid config values are:\n{desc}".format( dagster_type=repr(dagster_type), desc=VALID_CONFIG_DESC, )) if is_closed_python_optional_type(dagster_type): raise DagsterInvalidDefinitionError( "Cannot use typing.Optional as a config type. If you want this field to be " "optional, please use Field(<type>, is_required=False), and if you want this field to " "be required, but accept a value of None, use dagster.Noneable(<type>)." ) if is_typing_type(dagster_type): raise DagsterInvalidDefinitionError(( "You have passed in {dagster_type} to the config system. Types from " "the typing module in python are not allowed in the config system. " "You must use types that are imported from dagster or primitive types " "such as bool, int, etc.").format(dagster_type=dagster_type)) if dagster_type is List or isinstance(dagster_type, ListType): raise DagsterInvalidDefinitionError( "Cannot use List in the context of config. " + helpful_list_error_string()) if dagster_type is Set or isinstance(dagster_type, _TypedPythonSet): raise DagsterInvalidDefinitionError( "Cannot use Set in the context of a config field. " + helpful_list_error_string()) if dagster_type is Tuple or isinstance(dagster_type, _TypedPythonTuple): raise DagsterInvalidDefinitionError( "Cannot use Tuple in the context of a config field. " + helpful_list_error_string()) if isinstance(dagster_type, DagsterType): raise DagsterInvalidDefinitionError(( "You have passed an instance of DagsterType {type_name} to the config " "system (Repr of type: {dagster_type}). " "The DagsterType and config schema systems are separate. " "Valid config values are:\n{desc}").format( type_name=dagster_type.display_name, dagster_type=repr(dagster_type), desc=VALID_CONFIG_DESC, ), ) # If we are passed here either: # 1) We have been passed a python builtin # 2) We have been a dagster wrapping type that needs to be convert its config variant # e.g. dagster.List # 2) We have been passed an invalid thing. We return False to signify this. It is # up to callers to report a reasonable error. from dagster.primitive_mapping import ( remap_python_builtin_for_config, is_supported_config_python_builtin, ) if BuiltinEnum.contains(dagster_type): return ConfigType.from_builtin_enum(dagster_type) if is_supported_config_python_builtin(dagster_type): return remap_python_builtin_for_config(dagster_type) if dagster_type is None: return ConfigAnyInstance # This means that this is an error and we are return False to a callsite # We do the error reporting there because those callsites have more context return False
def is_callable_valid_config_arg(config: Dict[str, Any]) -> bool: return BuiltinEnum.contains(config) or is_supported_config_python_builtin( config)
def is_callable_valid_config_arg(config): return BuiltinEnum.contains(config) or is_supported_config_python_builtin(config)
def from_builtin_enum(builtin_enum: typing.Any) -> "ConfigType": check.invariant(BuiltinEnum.contains(builtin_enum), "param must be member of BuiltinEnum") return _CONFIG_MAP[builtin_enum]