def test_nullable_list_combos_coerciion():
list_of_int = resolve_to_runtime_type(List[Int])
assert_failure(list_of_int, None)
assert_success(list_of_int, [])
assert_success(list_of_int, [1])
assert_failure(list_of_int, [None])
nullable_int_of_list = resolve_to_runtime_type(Optional[List[Int]])
assert_success(nullable_int_of_list, None)
assert_success(nullable_int_of_list, [])
assert_success(nullable_int_of_list, [1])
assert_failure(nullable_int_of_list, [None])
list_of_nullable_int = resolve_to_runtime_type(List[Optional[Int]])
assert_failure(list_of_nullable_int, None)
assert_success(list_of_nullable_int, [])
assert_success(list_of_nullable_int, [1])
assert_success(list_of_nullable_int, [None])
nullable_list_of_nullable_int = resolve_to_runtime_type(
Optional[List[Optional[Int]]])
assert_success(nullable_list_of_nullable_int, None)
assert_success(nullable_list_of_nullable_int, [])
assert_success(nullable_list_of_nullable_int, [1])
assert_success(nullable_list_of_nullable_int, [None])
def test_file_system_intermediate_store_with_composite_type_storage_plugin():
run_id = str(uuid.uuid4())
intermediate_store = build_fs_intermediate_store(
DagsterInstance.ephemeral().intermediates_directory,
run_id=run_id,
type_storage_plugin_registry=TypeStoragePluginRegistry([
(RuntimeString, FancyStringFilesystemTypeStoragePlugin)
]),
)
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(['hello'], context,
resolve_to_runtime_type(List[String]),
['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(['hello'], context,
resolve_to_runtime_type(
Optional[String]), ['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(
['hello'], context,
resolve_to_runtime_type(List[Optional[String]]), ['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(
['hello'], context,
resolve_to_runtime_type(Optional[List[String]]), ['obj_name'])
def test_display_name():
int_runtime = resolve_to_runtime_type(Int)
assert int_runtime.display_name == 'Int'
list_int_runtime = resolve_to_runtime_type(List[Int])
assert list_int_runtime.display_name == '[Int]'
list_list_int_runtime = resolve_to_runtime_type(List[List[Int]])
assert list_list_int_runtime.display_name == '[[Int]]'
list_nullable_int_runtime = resolve_to_runtime_type(List[Optional[Int]])
assert list_nullable_int_runtime.display_name == '[Int?]'
def test_nullable_int_coercion():
int_type = resolve_to_runtime_type(Int)
assert_type_check(int_type.type_check(1))
res = int_type.type_check(None)
assert not res.success
nullable_int_type = resolve_to_runtime_type(Optional[Int])
assert_type_check(nullable_int_type.type_check(1))
assert_type_check(nullable_int_type.type_check(None))
def test_make_dagster_type():
OverwriteNameTuple = as_dagster_type(collections.namedtuple('SomeNamedTuple', 'prop'))
runtime_type = resolve_to_runtime_type(OverwriteNameTuple)
assert runtime_type.name == 'SomeNamedTuple'
assert OverwriteNameTuple(prop='foo').prop == 'foo'
OverwriteNameTuple = as_dagster_type(
collections.namedtuple('SomeNamedTuple', 'prop'), name='OverwriteName'
)
runtime_type = resolve_to_runtime_type(OverwriteNameTuple)
assert runtime_type.name == 'OverwriteName'
assert OverwriteNameTuple(prop='foo').prop == 'foo'
def test_inner_types():
assert resolve_to_runtime_type(Int).inner_types == []
list_int_runtime = resolve_to_runtime_type(List[Int])
assert inner_type_key_set(list_int_runtime) == set(['Int'])
list_list_int_runtime = resolve_to_runtime_type(List[List[Int]])
assert inner_type_key_set(list_list_int_runtime) == set(
['Int', 'List.Int'])
list_nullable_int_runtime = resolve_to_runtime_type(List[Optional[Int]])
assert inner_type_key_set(list_nullable_int_runtime) == set(
['Int', 'Optional.Int'])
assert not list_nullable_int_runtime.is_scalar
def test_dagster_type_decorator():
@dagster_type(name=None)
class Foo(object):
pass
@dagster_type()
class Bar(object):
pass
@dagster_type
class Baaz(object):
pass
assert resolve_to_runtime_type(Foo).name == 'Foo'
assert resolve_to_runtime_type(Bar).name == 'Bar'
assert resolve_to_runtime_type(Baaz).name == 'Baaz'
def test_file_system_intermediate_store_composite_types():
run_id = str(uuid.uuid4())
instance = DagsterInstance.ephemeral()
intermediate_store = build_fs_intermediate_store(
instance.intermediates_directory, run_id=run_id)
with yield_empty_pipeline_context(instance=instance,
run_id=run_id) as context:
intermediate_store.set_object([True, False], context,
resolve_to_runtime_type(List[Bool]),
['bool'])
assert intermediate_store.has_object(context, ['bool'])
assert intermediate_store.get_object(
context, resolve_to_runtime_type(List[Bool]),
['bool']).obj == [True, False]
def test_nullable_python_object_type():
nullable_type_bar = resolve_to_runtime_type(Optional[Bar])
assert_type_check(nullable_type_bar.type_check(BarObj()))
assert_type_check(nullable_type_bar.type_check(None))
res = nullable_type_bar.type_check('not_a_bar')
assert not res.success
def __init__(self, name, dagster_type=None, description=None):
''
self._name = check_valid_name(name)
self._runtime_type = check.inst(resolve_to_runtime_type(dagster_type),
RuntimeType)
self._description = check.opt_str_param(description, 'description')
def test_file_system_intermediate_store_composite_types_with_custom_serializer_for_inner_type(
):
run_id = str(uuid.uuid4())
instance = DagsterInstance.ephemeral()
intermediate_store = build_fs_intermediate_store(
instance.intermediates_directory, run_id=run_id)
with yield_empty_pipeline_context(run_id=run_id,
instance=instance) as context:
intermediate_store.set_object(['foo', 'bar'], context,
resolve_to_runtime_type(
List[LowercaseString]), ['list'])
assert intermediate_store.has_object(context, ['list'])
assert intermediate_store.get_object(
context, resolve_to_runtime_type(List[Bool]),
['list']).obj == ['foo', 'bar']
def test_dagster_type_decorator_name_desc():
@dagster_type(name='DifferentName', description='desc')
class Something(object):
pass
runtime_type = resolve_to_runtime_type(Something)
assert runtime_type.name == 'DifferentName'
assert runtime_type.description == 'desc'
def remap_python_builtin_for_runtime(ttype):
'''This function remaps a python type to a Dagster type, or passes it through if it cannot be
remapped.
'''
from dagster.core.types.runtime_type import resolve_to_runtime_type
check.param_invariant(is_supported_runtime_python_builtin(ttype), 'ttype')
return resolve_to_runtime_type(SUPPORTED_RUNTIME_BUILTINS[ttype])
def get_intermediate(self,
context,
step_key,
dagster_type,
output_name='result'):
return self.get_object(
context=context,
runtime_type=resolve_to_runtime_type(dagster_type),
paths=self.paths_for_intermediate(step_key, output_name),
)
def test_gcs_intermediate_store_composite_types_with_custom_serializer_for_inner_type(
gcs_bucket):
run_id = str(uuid.uuid4())
intermediate_store = GCSIntermediateStore(run_id=run_id,
gcs_bucket=gcs_bucket)
with yield_empty_pipeline_context(run_id=run_id) as context:
try:
intermediate_store.set_object(
['foo', 'bar'],
context,
resolve_to_runtime_type(List[LowercaseString]),
['list'],
)
assert intermediate_store.has_object(context, ['list'])
assert intermediate_store.get_object(
context, resolve_to_runtime_type(List[Bool]),
['list']).obj == ['foo', 'bar']
finally:
intermediate_store.rm_object(context, ['foo'])
def test_python_mapping():
runtime = resolve_to_runtime_type(int)
assert runtime.name == 'Int'
runtime = resolve_to_runtime_type(str)
assert runtime.name == 'String'
runtime = resolve_to_runtime_type(bool)
assert runtime.name == 'Bool'
runtime = resolve_to_runtime_type(float)
assert runtime.name == 'Float'
@lambda_solid(input_defs=[InputDefinition('num', int)])
def add_one(num):
return num + 1
assert add_one.input_defs[0].runtime_type.name == 'Int'
runtime = resolve_to_runtime_type(float)
runtime.type_check(1.0)
res = runtime.type_check(1)
assert not res.success
runtime = resolve_to_runtime_type(bool)
runtime.type_check(True)
res = runtime.type_check(1)
assert not res.success
def test_runtime_optional_set():
set_runtime_type = resolve_to_runtime_type(
Optional[create_typed_runtime_set(int)])
set_runtime_type.type_check({1})
set_runtime_type.type_check(set())
set_runtime_type.type_check(None)
res = set_runtime_type.type_check('nope')
assert not res.success
res = set_runtime_type.type_check({'nope'})
assert not res.success
def test_gcs_intermediate_store_with_composite_type_storage_plugin(gcs_bucket):
run_id = str(uuid.uuid4())
intermediate_store = GCSIntermediateStore(
run_id=run_id,
gcs_bucket=gcs_bucket,
type_storage_plugin_registry=TypeStoragePluginRegistry([
(RuntimeString, FancyStringGCSTypeStoragePlugin)
]),
)
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(['hello'], context,
resolve_to_runtime_type(List[String]),
['obj_name'])
def check_dagster_type(dagster_type, value):
'''Test a custom Dagster type.
Args:
dagster_type (Any): The Dagster type to test. Should be one of the
:ref:`built-in types <builtin>`, a dagster type explicitly constructed with
:py:func:`as_dagster_type`, :py:func:`@dagster_type <dagster_type>`, or
:py:func:`define_python_dagster_type`, or a Python type.
value (Any): The runtime value to test.
Returns:
TypeCheck: The result of the type check.
Examples:
.. code-block:: python
assert check_dagster_type(Dict[Any, Any], {'foo': 'bar'}).success
'''
if is_typing_type(dagster_type):
raise DagsterInvariantViolationError(
(
'Must pass in a type from dagster module. You passed {dagster_type} '
'which is part of python\'s typing module.'
).format(dagster_type=dagster_type)
)
runtime_type = resolve_to_runtime_type(dagster_type)
type_check = runtime_type.type_check(value)
if not isinstance(type_check, TypeCheck):
raise DagsterInvariantViolationError(
'Type checks can only return TypeCheck. Type {type_name} returned {value}.'.format(
type_name=runtime_type.name, value=repr(type_check)
)
)
return type_check
def __init__(self, dagster_type=None, name=None, description=None, is_optional=False):
self._name = check_valid_name(check.opt_str_param(name, 'name', DEFAULT_OUTPUT))
self._runtime_type = resolve_to_runtime_type(dagster_type)
self._description = check.opt_str_param(description, 'description')
self._optional = check.bool_param(is_optional, 'is_optional')
def test_comparison():
# Base types
assert resolve_to_runtime_type(Any) == resolve_to_runtime_type(Any)
assert resolve_to_runtime_type(String) == resolve_to_runtime_type(String)
assert resolve_to_runtime_type(Bool) == resolve_to_runtime_type(Bool)
assert resolve_to_runtime_type(Float) == resolve_to_runtime_type(Float)
assert resolve_to_runtime_type(Int) == resolve_to_runtime_type(Int)
assert resolve_to_runtime_type(Path) == resolve_to_runtime_type(Path)
assert resolve_to_runtime_type(String) == resolve_to_runtime_type(String)
assert resolve_to_runtime_type(Nothing) == resolve_to_runtime_type(Nothing)
assert resolve_to_runtime_type(
Optional[String]) == resolve_to_runtime_type(Optional[String])
types = [Any, Bool, Float, Int, Path, String, Nothing]
non_equal_pairs = [(t1, t2) for t1 in types for t2 in types if t1 != t2]
for t1, t2 in non_equal_pairs:
assert resolve_to_runtime_type(t1) != resolve_to_runtime_type(t2)
assert resolve_to_runtime_type(
Optional[String]) != resolve_to_runtime_type(Optional[Int])
# List type
assert resolve_to_runtime_type(List) == resolve_to_runtime_type(List)
assert resolve_to_runtime_type(List[String]) == resolve_to_runtime_type(
List[String])
assert resolve_to_runtime_type(List[List[Int]]) == resolve_to_runtime_type(
List[List[Int]])
assert resolve_to_runtime_type(
List[Optional[String]]) == resolve_to_runtime_type(
List[Optional[String]])
assert resolve_to_runtime_type(List[String]) != resolve_to_runtime_type(
List[Int])
assert resolve_to_runtime_type(
List[List[String]]) != resolve_to_runtime_type(List[List[Int]])
assert resolve_to_runtime_type(List[String]) != resolve_to_runtime_type(
List[Optional[String]])
# Tuple type
assert resolve_to_runtime_type(Tuple) == resolve_to_runtime_type(Tuple)
assert resolve_to_runtime_type(
Tuple[String, Int]) == resolve_to_runtime_type(Tuple[String, Int])
assert resolve_to_runtime_type(
Tuple[Tuple[String,
Int]]) == resolve_to_runtime_type(Tuple[Tuple[String,
Int]])
assert resolve_to_runtime_type(Tuple[Optional[String],
Int]) == resolve_to_runtime_type(
Tuple[Optional[String], Int])
assert resolve_to_runtime_type(
Tuple[String, Int]) != resolve_to_runtime_type(Tuple[Int, String])
assert resolve_to_runtime_type(
Tuple[Tuple[String, Int]]) != resolve_to_runtime_type(
Tuple[Tuple[Int, String]])
assert resolve_to_runtime_type(Tuple[String]) != resolve_to_runtime_type(
Tuple[Optional[String]])
# Set type
assert resolve_to_runtime_type(Set) == resolve_to_runtime_type(Set)
assert resolve_to_runtime_type(Set[String]) == resolve_to_runtime_type(
Set[String])
assert resolve_to_runtime_type(Set[Set[Int]]) == resolve_to_runtime_type(
Set[Set[Int]])
assert resolve_to_runtime_type(
Set[Optional[String]]) == resolve_to_runtime_type(
Set[Optional[String]])
assert resolve_to_runtime_type(Set[String]) != resolve_to_runtime_type(
Set[Int])
assert resolve_to_runtime_type(
Set[Set[String]]) != resolve_to_runtime_type(Set[Set[Int]])
assert resolve_to_runtime_type(Set[String]) != resolve_to_runtime_type(
Set[Optional[String]])
# Dict type
assert resolve_to_runtime_type(Dict) == resolve_to_runtime_type(Dict)
assert resolve_to_runtime_type(
Dict[String, Int]) == resolve_to_runtime_type(Dict[String, Int])
assert resolve_to_runtime_type(
Dict[String, Dict[String,
Int]]) == resolve_to_runtime_type(Dict[String,
Dict[String,
Int]])
assert resolve_to_runtime_type(
Dict[String, Int]) != resolve_to_runtime_type(Dict[Int, String])
assert resolve_to_runtime_type(
Dict[Int, Dict[String, Int]]) != resolve_to_runtime_type(
Dict[String, Dict[String, Int]])
def test_make_dagster_type_from_builtin():
OrderedDict = as_dagster_type(collections.OrderedDict)
assert OrderedDict is collections.OrderedDict
assert OrderedDict([('foo', 'bar')]) == collections.OrderedDict([('foo', 'bar')])
assert isinstance(resolve_to_runtime_type(OrderedDict), RuntimeType)
assert resolve_to_runtime_type(OrderedDict).python_type is collections.OrderedDict
def test_is_any():
assert not resolve_to_runtime_type(Int).is_any
assert resolve_to_runtime_type(Int).is_scalar
