def test_file_system_intermediate_store_with_composite_type_storage_plugin():
run_id = str(uuid.uuid4())
intermediate_store = build_fs_intermediate_store(
DagsterInstance.ephemeral().intermediates_directory,
run_id=run_id,
type_storage_plugin_registry=TypeStoragePluginRegistry(
{RuntimeString: FancyStringFilesystemTypeStoragePlugin}),
)
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(['hello'], context,
resolve_to_runtime_type(List[String]),
['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(['hello'], context,
resolve_to_runtime_type(
Optional[String]), ['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(
['hello'], context,
resolve_to_runtime_type(List[Optional[String]]), ['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(
['hello'], context,
resolve_to_runtime_type(Optional[List[String]]), ['obj_name'])
def test_nullable_list_combos_coerciion():
list_of_int = resolve_to_runtime_type(List[Int])
assert_failure(list_of_int, None)
assert_success(list_of_int, [])
assert_success(list_of_int, [1])
assert_failure(list_of_int, [None])
nullable_int_of_list = resolve_to_runtime_type(Optional[List[Int]])
assert_success(nullable_int_of_list, None)
assert_success(nullable_int_of_list, [])
assert_success(nullable_int_of_list, [1])
assert_failure(nullable_int_of_list, [None])
list_of_nullable_int = resolve_to_runtime_type(List[Optional[Int]])
assert_failure(list_of_nullable_int, None)
assert_success(list_of_nullable_int, [])
assert_success(list_of_nullable_int, [1])
assert_success(list_of_nullable_int, [None])
nullable_list_of_nullable_int = resolve_to_runtime_type(Optional[List[Optional[Int]]])
assert_success(nullable_list_of_nullable_int, None)
assert_success(nullable_list_of_nullable_int, [])
assert_success(nullable_list_of_nullable_int, [1])
assert_success(nullable_list_of_nullable_int, [None])
def test_make_dagster_type_from_builtin():
OrderedDict = as_dagster_type(collections.OrderedDict)
assert OrderedDict is collections.OrderedDict
assert OrderedDict([('foo', 'bar')]) == collections.OrderedDict([('foo',
'bar')])
assert isinstance(resolve_to_runtime_type(OrderedDict), RuntimeType)
assert resolve_to_runtime_type(
OrderedDict).python_type is collections.OrderedDict
def test_display_name():
int_runtime = resolve_to_runtime_type(Int)
assert int_runtime.display_name == 'Int'
list_int_runtime = resolve_to_runtime_type(List[Int])
assert list_int_runtime.display_name == '[Int]'
list_list_int_runtime = resolve_to_runtime_type(List[List[Int]])
assert list_list_int_runtime.display_name == '[[Int]]'
list_nullable_int_runtime = resolve_to_runtime_type(List[Optional[Int]])
assert list_nullable_int_runtime.display_name == '[Int?]'
def test_nullable_int_coercion():
int_type = resolve_to_runtime_type(Int)
assert_type_check(int_type.type_check(1))
res = int_type.type_check(None)
assert not res.success
nullable_int_type = resolve_to_runtime_type(Optional[Int])
assert_type_check(nullable_int_type.type_check(1))
assert_type_check(nullable_int_type.type_check(None))
def test_make_dagster_type():
OverwriteNameTuple = as_dagster_type(
collections.namedtuple('SomeNamedTuple', 'prop'))
runtime_type = resolve_to_runtime_type(OverwriteNameTuple)
assert runtime_type.name == 'SomeNamedTuple'
assert OverwriteNameTuple(prop='foo').prop == 'foo'
OverwriteNameTuple = as_dagster_type(collections.namedtuple(
'SomeNamedTuple', 'prop'),
name='OverwriteName')
runtime_type = resolve_to_runtime_type(OverwriteNameTuple)
assert runtime_type.name == 'OverwriteName'
assert OverwriteNameTuple(prop='foo').prop == 'foo'
def test_inner_types():
assert resolve_to_runtime_type(Int).inner_types == []
list_int_runtime = resolve_to_runtime_type(List[Int])
assert inner_type_key_set(list_int_runtime) == set(['Int'])
list_list_int_runtime = resolve_to_runtime_type(List[List[Int]])
assert inner_type_key_set(list_list_int_runtime) == set(
['Int', 'List.Int'])
list_nullable_int_runtime = resolve_to_runtime_type(List[Optional[Int]])
assert inner_type_key_set(list_nullable_int_runtime) == set(
['Int', 'Optional.Int'])
assert not list_nullable_int_runtime.is_scalar
def check_dagster_type(dagster_type, value):
'''Test a custom Dagster type.
Args:
dagster_type (Any): The Dagster type to test. Should be one of the
:ref:`built-in types <builtin>`, a dagster type explicitly constructed with
:py:func:`as_dagster_type`, :py:func:`@dagster_type <dagster_type>`, or
:py:func:`define_python_dagster_type`, or a Python type.
value (Any): The runtime value to test.
Returns:
TypeCheck: The result of the type check.
Examples:
.. code-block:: python
assert check_dagster_type(Dict[Any, Any], {'foo': 'bar'}).success
'''
if is_typing_type(dagster_type):
raise DagsterInvariantViolationError((
'Must pass in a type from dagster module. You passed {dagster_type} '
'which is part of python\'s typing module.').format(
dagster_type=dagster_type))
runtime_type = resolve_to_runtime_type(dagster_type)
type_check = runtime_type.type_check(value)
if not isinstance(type_check, TypeCheck):
raise DagsterInvariantViolationError(
'Type checks can only return TypeCheck. Type {type_name} returned {value}.'
.format(type_name=runtime_type.name, value=repr(type_check)))
return type_check
def test_dagster_type_decorator():
@dagster_type(name=None)
class Foo(object):
pass
@dagster_type()
class Bar(object):
pass
@dagster_type
class Baaz(object):
pass
assert resolve_to_runtime_type(Foo).name == 'Foo'
assert resolve_to_runtime_type(Bar).name == 'Bar'
assert resolve_to_runtime_type(Baaz).name == 'Baaz'
def test_file_system_intermediate_store_composite_types():
run_id = str(uuid.uuid4())
instance = DagsterInstance.ephemeral()
intermediate_store = build_fs_intermediate_store(
instance.intermediates_directory, run_id=run_id)
with yield_empty_pipeline_context(instance=instance,
run_id=run_id) as context:
intermediate_store.set_object([True, False], context,
resolve_to_runtime_type(List[Bool]),
['bool'])
assert intermediate_store.has_object(context, ['bool'])
assert intermediate_store.get_object(
context, resolve_to_runtime_type(List[Bool]),
['bool']).obj == [True, False]
def test_dagster_type_decorator_name_desc():
@dagster_type(name='DifferentName', description='desc')
class Something(object):
pass
runtime_type = resolve_to_runtime_type(Something)
assert runtime_type.name == 'DifferentName'
assert runtime_type.description == 'desc'
def __init__(self, name, dagster_type=None, description=None):
''
self._name = check_valid_name(name)
self._runtime_type = check.inst(resolve_to_runtime_type(dagster_type),
RuntimeType)
self._description = check.opt_str_param(description, 'description')
def test_file_system_intermediate_store_composite_types_with_custom_serializer_for_inner_type(
):
run_id = str(uuid.uuid4())
instance = DagsterInstance.ephemeral()
intermediate_store = build_fs_intermediate_store(
instance.intermediates_directory, run_id=run_id)
with yield_empty_pipeline_context(run_id=run_id,
instance=instance) as context:
intermediate_store.set_object(['foo', 'bar'], context,
resolve_to_runtime_type(
List[LowercaseString]), ['list'])
assert intermediate_store.has_object(context, ['list'])
assert intermediate_store.get_object(
context, resolve_to_runtime_type(List[Bool]),
['list']).obj == ['foo', 'bar']
def test_nullable_python_object_type():
nullable_type_bar = resolve_to_runtime_type(Optional[Bar])
assert_type_check(nullable_type_bar.type_check(BarObj()))
assert_type_check(nullable_type_bar.type_check(None))
res = nullable_type_bar.type_check('not_a_bar')
assert not res.success
def remap_python_builtin_for_runtime(ttype):
'''This function remaps a python type to a Dagster type, or passes it through if it cannot be
remapped.
'''
from dagster.core.types.runtime.runtime_type import resolve_to_runtime_type
check.param_invariant(is_supported_runtime_python_builtin(ttype), 'ttype')
return resolve_to_runtime_type(SUPPORTED_RUNTIME_BUILTINS[ttype])
def test_s3_intermediate_store_composite_types_with_custom_serializer_for_inner_type(
s3_bucket):
run_id = str(uuid.uuid4())
intermediate_store = S3IntermediateStore(run_id=run_id,
s3_bucket=s3_bucket)
with yield_empty_pipeline_context(run_id=run_id) as context:
try:
intermediate_store.set_object(
['foo', 'bar'],
context,
resolve_to_runtime_type(List[LowercaseString]),
['list'],
)
assert intermediate_store.has_object(context, ['list'])
assert intermediate_store.get_object(
context, resolve_to_runtime_type(List[Bool]),
['list']).obj == ['foo', 'bar']
finally:
intermediate_store.rm_object(context, ['foo'])
def test_python_mapping():
runtime = resolve_to_runtime_type(int)
assert runtime.name == 'Int'
runtime = resolve_to_runtime_type(str)
assert runtime.name == 'String'
runtime = resolve_to_runtime_type(bool)
assert runtime.name == 'Bool'
runtime = resolve_to_runtime_type(float)
assert runtime.name == 'Float'
@lambda_solid(input_defs=[InputDefinition('num', int)])
def add_one(num):
return num + 1
assert add_one.input_defs[0].runtime_type.name == 'Int'
runtime = resolve_to_runtime_type(float)
runtime.type_check(1.0)
res = runtime.type_check(1)
assert not res.success
runtime = resolve_to_runtime_type(bool)
runtime.type_check(True)
res = runtime.type_check(1)
assert not res.success
def test_runtime_optional_set():
set_runtime_type = resolve_to_runtime_type(
Optional[create_typed_runtime_set(int)])
set_runtime_type.type_check({1})
set_runtime_type.type_check(set())
set_runtime_type.type_check(None)
res = set_runtime_type.type_check('nope')
assert not res.success
res = set_runtime_type.type_check({'nope'})
assert not res.success
def test_s3_intermediate_store_with_composite_type_storage_plugin(s3_bucket):
run_id = str(uuid.uuid4())
intermediate_store = S3IntermediateStore(
run_id=run_id,
s3_bucket=s3_bucket,
type_storage_plugin_registry=TypeStoragePluginRegistry(
{RuntimeString: FancyStringS3TypeStoragePlugin}),
)
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(['hello'], context,
resolve_to_runtime_type(List[String]),
['obj_name'])
def get_intermediate(self, context, step_key, dagster_type, output_name='result'):
return self.get_object(
context=context,
runtime_type=resolve_to_runtime_type(dagster_type),
paths=self.paths_for_intermediate(step_key, output_name),
)
def __init__(self, dagster_type=None, name=None, description=None, is_optional=False):
self._name = check_valid_name(check.opt_str_param(name, 'name', DEFAULT_OUTPUT))
self._runtime_type = resolve_to_runtime_type(dagster_type)
self._description = check.opt_str_param(description, 'description')
self._optional = check.bool_param(is_optional, 'is_optional')
def test_is_any():
assert not resolve_to_runtime_type(Int).is_any
assert resolve_to_runtime_type(Int).is_scalar
