def test_file_system_intermediate_store_with_composite_type_storage_plugin():
run_id = str(uuid.uuid4())
intermediate_store = build_fs_intermediate_store(
DagsterInstance.ephemeral().intermediates_directory,
run_id=run_id,
type_storage_plugin_registry=TypeStoragePluginRegistry(
{RuntimeString.inst(): FancyStringFilesystemTypeStoragePlugin}),
)
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(['hello'], context,
resolve_to_runtime_type(List[String]),
['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(['hello'], context,
resolve_to_runtime_type(
Optional[String]), ['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(
['hello'], context,
resolve_to_runtime_type(List[Optional[String]]), ['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(
['hello'], context,
resolve_to_runtime_type(Optional[List[String]]), ['obj_name'])
def test_file_system_intermediate_store_composite_types_with_custom_serializer_for_inner_type(
):
run_id = str(uuid.uuid4())
intermediate_store = FileSystemIntermediateStore(run_id=run_id)
assert intermediate_store.root == os.path.join(
seven.get_system_temp_directory(), 'dagster', 'runs', run_id, 'files')
with yield_empty_pipeline_context(run_id=run_id) as context:
try:
intermediate_store.set_object(
['foo', 'bar'],
context,
resolve_to_runtime_type(List(LowercaseString)).inst(),
['list'],
)
assert intermediate_store.has_object(context, ['list'])
assert intermediate_store.get_object(
context,
resolve_to_runtime_type(List(Bool)).inst(),
['list']) == ['foo', 'bar']
finally:
try:
shutil.rmtree(intermediate_store.root)
except seven.FileNotFoundError:
pass
def test_file_system_intermediate_store_with_composite_type_storage_plugin():
run_id = str(uuid.uuid4())
# FIXME need a dedicated test bucket
intermediate_store = FileSystemIntermediateStore(
run_id=run_id,
type_storage_plugin_registry=TypeStoragePluginRegistry(
{RuntimeString.inst(): FancyStringFilesystemTypeStoragePlugin}),
)
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(['hello'], context,
resolve_to_runtime_type(List[String]),
['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(['hello'], context,
resolve_to_runtime_type(
Optional[String]), ['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(
['hello'], context,
resolve_to_runtime_type(List[Optional[String]]), ['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(
['hello'], context,
resolve_to_runtime_type(Optional[List[String]]), ['obj_name'])
def test_file_system_object_store_composite_types():
run_id = str(uuid.uuid4())
object_store = FileSystemObjectStore(run_id=run_id)
assert object_store.root == os.path.join(seven.get_system_temp_directory(),
'dagster', 'runs', run_id,
'files')
with yield_pipeline_execution_context(PipelineDefinition([]), {},
RunConfig(run_id=run_id)) as context:
try:
object_store.set_object([True, False], context,
resolve_to_runtime_type(
List_(Bool_)).inst(), ['bool'])
assert object_store.has_object(context, ['bool'])
assert object_store.get_object(
context,
resolve_to_runtime_type(List_(Bool_)).inst(),
['bool']) == [True, False]
finally:
try:
shutil.rmtree(object_store.root)
except seven.FileNotFoundError:
pass
def test_file_system_object_store_composite_types_with_custom_serializer_for_inner_type(
):
run_id = str(uuid.uuid4())
object_store = FileSystemObjectStore(run_id=run_id)
assert object_store.root == os.path.join(seven.get_system_temp_directory(),
'dagster', 'runs', run_id,
'files')
with yield_pipeline_execution_context(PipelineDefinition([]), {},
RunConfig(run_id=run_id)) as context:
try:
object_store.set_object(
['foo', 'bar'],
context,
resolve_to_runtime_type(List_(LowercaseString)).inst(),
['list'],
)
assert object_store.has_object(context, ['list'])
assert object_store.get_object(
context,
resolve_to_runtime_type(List_(Bool_)).inst(),
['list']) == ['foo', 'bar']
finally:
try:
shutil.rmtree(object_store.root)
except seven.FileNotFoundError:
pass
def test_nullable_list_combos_coerciion():
list_of_int = resolve_to_runtime_type(List(Int))
assert_failure(list_of_int.coerce_runtime_value, None)
assert_success(list_of_int.coerce_runtime_value, [])
assert_success(list_of_int.coerce_runtime_value, [1])
assert_failure(list_of_int.coerce_runtime_value, [None])
nullable_int_of_list = resolve_to_runtime_type(Nullable(List(Int)))
assert_success(nullable_int_of_list.coerce_runtime_value, None)
assert_success(nullable_int_of_list.coerce_runtime_value, [])
assert_success(nullable_int_of_list.coerce_runtime_value, [1])
assert_failure(nullable_int_of_list.coerce_runtime_value, [None])
list_of_nullable_int = resolve_to_runtime_type(List(Nullable(Int)))
assert_failure(list_of_nullable_int.coerce_runtime_value, None)
assert_success(list_of_nullable_int.coerce_runtime_value, [])
assert_success(list_of_nullable_int.coerce_runtime_value, [1])
assert_success(list_of_nullable_int.coerce_runtime_value, [None])
nullable_list_of_nullable_int = resolve_to_runtime_type(Nullable(List(Nullable(Int))))
assert_success(nullable_list_of_nullable_int.coerce_runtime_value, None)
assert_success(nullable_list_of_nullable_int.coerce_runtime_value, [])
assert_success(nullable_list_of_nullable_int.coerce_runtime_value, [1])
assert_success(nullable_list_of_nullable_int.coerce_runtime_value, [None])
def test_nullable_list_combos_coerciion():
list_of_int = resolve_to_runtime_type(List[Int])
assert_failure(list_of_int, None)
assert_success(list_of_int, [])
assert_success(list_of_int, [1])
assert_failure(list_of_int, [None])
nullable_int_of_list = resolve_to_runtime_type(Optional[List[Int]])
assert_success(nullable_int_of_list, None)
assert_success(nullable_int_of_list, [])
assert_success(nullable_int_of_list, [1])
assert_failure(nullable_int_of_list, [None])
list_of_nullable_int = resolve_to_runtime_type(List[Optional[Int]])
assert_failure(list_of_nullable_int, None)
assert_success(list_of_nullable_int, [])
assert_success(list_of_nullable_int, [1])
assert_success(list_of_nullable_int, [None])
nullable_list_of_nullable_int = resolve_to_runtime_type(
Optional[List[Optional[Int]]])
assert_success(nullable_list_of_nullable_int, None)
assert_success(nullable_list_of_nullable_int, [])
assert_success(nullable_list_of_nullable_int, [1])
assert_success(nullable_list_of_nullable_int, [None])
def test_make_dagster_type_from_builtin():
OrderedDict = as_dagster_type(collections.OrderedDict)
assert OrderedDict is collections.OrderedDict
assert OrderedDict([('foo', 'bar')]) == collections.OrderedDict([('foo',
'bar')])
assert isinstance(resolve_to_runtime_type(OrderedDict), RuntimeType)
assert resolve_to_runtime_type(
OrderedDict).python_type is collections.OrderedDict
def test_nullable_int_coercion():
int_type = resolve_to_runtime_type(Int)
assert_type_check(int_type.type_check(1))
with pytest.raises(Failure):
int_type.type_check(None)
nullable_int_type = resolve_to_runtime_type(Optional[Int])
assert_type_check(nullable_int_type.type_check(1))
assert_type_check(nullable_int_type.type_check(None))
def test_display_name():
int_runtime = resolve_to_runtime_type(Int)
assert int_runtime.display_name == 'Int'
list_int_runtime = resolve_to_runtime_type(List[Int])
assert list_int_runtime.display_name == '[Int]'
list_list_int_runtime = resolve_to_runtime_type(List[List[Int]])
assert list_list_int_runtime.display_name == '[[Int]]'
list_nullable_int_runtime = resolve_to_runtime_type(List[Optional[Int]])
assert list_nullable_int_runtime.display_name == '[Int?]'
def test_nullable_int_coercion():
int_type = resolve_to_runtime_type(Int)
assert int_type.coerce_runtime_value(1) == 1
with pytest.raises(DagsterRuntimeCoercionError):
assert int_type.coerce_runtime_value(None)
nullable_int_type = resolve_to_runtime_type(Nullable(Int))
assert nullable_int_type.coerce_runtime_value(1) == 1
assert nullable_int_type.coerce_runtime_value(None) is None
def test_display_name():
int_runtime = resolve_to_runtime_type(Int)
assert int_runtime.display_name == 'Int'
list_int_runtime = resolve_to_runtime_type(List(Int))
assert list_int_runtime.display_name == '[Int]'
list_list_int_runtime = resolve_to_runtime_type(List(List(Int)))
assert list_list_int_runtime.display_name == '[[Int]]'
list_nullable_int_runtime = resolve_to_runtime_type(List(Nullable(Int)))
assert list_nullable_int_runtime.display_name == '[Int?]'
def test_inner_types():
assert resolve_to_runtime_type(Int).inner_types == []
list_int_runtime = resolve_to_runtime_type(List[Int])
assert inner_type_key_set(list_int_runtime) == set(['Int'])
list_list_int_runtime = resolve_to_runtime_type(List[List[Int]])
assert inner_type_key_set(list_list_int_runtime) == set(['Int', 'List.Int'])
list_nullable_int_runtime = resolve_to_runtime_type(List[Optional[Int]])
assert inner_type_key_set(list_nullable_int_runtime) == set(['Int', 'Optional.Int'])
def test_make_dagster_type():
OverwriteNameTuple = as_dagster_type(
collections.namedtuple('SomeNamedTuple', 'prop'))
runtime_type = resolve_to_runtime_type(OverwriteNameTuple)
assert runtime_type.name == 'SomeNamedTuple'
assert OverwriteNameTuple(prop='foo').prop == 'foo'
OverwriteNameTuple = as_dagster_type(collections.namedtuple(
'SomeNamedTuple', 'prop'),
name='OverwriteName')
runtime_type = resolve_to_runtime_type(OverwriteNameTuple)
assert runtime_type.name == 'OverwriteName'
assert OverwriteNameTuple(prop='foo').prop == 'foo'
def test_inner_types():
assert resolve_to_runtime_type(Int).inner_types == []
list_int_runtime = resolve_to_runtime_type(List(Int))
assert inner_type_key_set(list_int_runtime) == set(['Int'])
list_list_int_runtime = resolve_to_runtime_type(List(List(Int)))
assert inner_type_key_set(list_list_int_runtime) == set(
['Int', 'List.Int'])
list_nullable_int_runtime = resolve_to_runtime_type(List(Nullable(Int)))
assert inner_type_key_set(list_nullable_int_runtime) == set(
['Int', 'Nullable.Int'])
def test_python_mapping():
int_runtime = resolve_to_runtime_type(int)
assert int_runtime.name == 'Int'
int_runtime = resolve_to_runtime_type(str)
assert int_runtime.name == 'String'
int_runtime = resolve_to_runtime_type(bool)
assert int_runtime.name == 'Bool'
int_runtime = resolve_to_runtime_type(float)
assert int_runtime.name == 'Float'
@lambda_solid(inputs=[InputDefinition('num', int)])
def add_one(num):
return num + 1
assert add_one.input_defs[0].runtime_type.name == 'Int'
def __init__(self, name, dagster_type=None, description=None):
''
self.name = check_valid_name(name)
self.runtime_type = check.inst(resolve_to_runtime_type(dagster_type), RuntimeType)
self.description = check.opt_str_param(description, 'description')
def check_dagster_type(dagster_type, value):
'''Test a custom Dagster type.
Args:
dagster_type (Any): The Dagster type to test. Should be one of the
:ref:`built-in types <builtin>`, a dagster type explicitly constructed with
:py:func:`as_dagster_type`, :py:func:`@dagster_type <dagster_type>`, or
:py:func:`define_python_dagster_type`, or a Python type.
value (Any): The runtime value to test.
Returns:
TypeCheck: The result of the type check.
**Examples**:
.. code-block:: python
assert check_dagster_type(Dict[Any, Any], {'foo': 'bar'}).success
'''
runtime_type = resolve_to_runtime_type(dagster_type)
type_check = runtime_type.type_check(value)
if not isinstance(type_check, TypeCheck):
raise DagsterInvariantViolationError(
('Type checks can only return TypeCheck. Type '
'{type_name} returned {value}.').format(
type_name=runtime_type.name, value=repr(type_check)))
return type_check
def test_dagster_type_decorator():
@dagster_type(name=None)
class Foo(object):
pass
@dagster_type()
class Bar(object):
pass
@dagster_type
class Baaz(object):
pass
assert resolve_to_runtime_type(Foo).name == 'Foo'
assert resolve_to_runtime_type(Bar).name == 'Bar'
assert resolve_to_runtime_type(Baaz).name == 'Baaz'
def test_nullable_python_object_type():
nullable_type_bar = resolve_to_runtime_type(Optional[Bar])
assert_type_check(nullable_type_bar.type_check(BarObj()))
assert_type_check(nullable_type_bar.type_check(None))
with pytest.raises(Failure):
nullable_type_bar.type_check('not_a_bar')
def test_dagster_type_decorator_name_desc():
@dagster_type(name='DifferentName', description='desc')
class Something(object):
pass
runtime_type = resolve_to_runtime_type(Something)
assert runtime_type.name == 'DifferentName'
assert runtime_type.description == 'desc'
def test_file_system_intermediate_store_composite_types():
run_id = str(uuid.uuid4())
instance = DagsterInstance.ephemeral()
intermediate_store = FilesystemIntermediateStore.for_instance(
instance=instance, run_id=run_id)
with yield_empty_pipeline_context(instance=instance,
run_id=run_id) as context:
intermediate_store.set_object([True, False], context,
resolve_to_runtime_type(
List[Bool]).inst(), ['bool'])
assert intermediate_store.has_object(context, ['bool'])
assert intermediate_store.get_object(
context,
resolve_to_runtime_type(List[Bool]).inst(),
['bool']).obj == [True, False]
def test_nullable_python_object_type():
nullable_type_bar = resolve_to_runtime_type(Nullable(Bar))
assert nullable_type_bar.coerce_runtime_value(BarObj())
assert nullable_type_bar.coerce_runtime_value(None) is None
with pytest.raises(DagsterRuntimeCoercionError):
nullable_type_bar.coerce_runtime_value('not_a_bar')
def test_file_system_intermediate_store_composite_types_with_custom_serializer_for_inner_type(
):
run_id = str(uuid.uuid4())
instance = DagsterInstance.ephemeral()
intermediate_store = FilesystemIntermediateStore.for_instance(
instance, run_id=run_id)
with yield_empty_pipeline_context(run_id=run_id,
instance=instance) as context:
intermediate_store.set_object(
['foo', 'bar'], context,
resolve_to_runtime_type(List[LowercaseString]).inst(), ['list'])
assert intermediate_store.has_object(context, ['list'])
assert intermediate_store.get_object(
context,
resolve_to_runtime_type(List[Bool]).inst(),
['list']).obj == ['foo', 'bar']
def __init__(self, name, dagster_type=None, expectations=None, description=None):
''
self.name = check_valid_name(name)
self.runtime_type = check.inst(resolve_to_runtime_type(dagster_type), RuntimeType)
self.expectations = check.opt_list_param(
expectations, 'expectations', of_type=IOExpectationDefinition
)
self.description = check.opt_str_param(description, 'description')
def get_intermediate(self,
context,
step_key,
dagster_type,
output_name='result'):
return self.get_object(
context=context,
runtime_type=resolve_to_runtime_type(dagster_type),
paths=self.paths_for_intermediate(step_key, output_name),
)
def test_s3_intermediate_store_composite_types_with_custom_serializer_for_inner_type(s3_bucket):
run_id = str(uuid.uuid4())
intermediate_store = S3IntermediateStore(run_id=run_id, s3_bucket=s3_bucket)
with yield_empty_pipeline_context(run_id=run_id) as context:
try:
intermediate_store.set_object(
['foo', 'bar'],
context,
resolve_to_runtime_type(List[LowercaseString]).inst(),
['list'],
)
assert intermediate_store.has_object(context, ['list'])
assert intermediate_store.get_object(
context, resolve_to_runtime_type(List[Bool]).inst(), ['list']
).obj == ['foo', 'bar']
finally:
intermediate_store.rm_object(context, ['foo'])
def __init__(self,
dagster_type=None,
name=None,
description=None,
is_optional=False):
self._name = check_valid_name(
check.opt_str_param(name, 'name', DEFAULT_OUTPUT))
self._runtime_type = resolve_to_runtime_type(dagster_type)
self._description = check.opt_str_param(description, 'description')
self._optional = check.bool_param(is_optional, 'is_optional')
def test_python_mapping():
runtime = resolve_to_runtime_type(int)
assert runtime.name == 'Int'
runtime = resolve_to_runtime_type(str)
assert runtime.name == 'String'
runtime = resolve_to_runtime_type(bool)
assert runtime.name == 'Bool'
runtime = resolve_to_runtime_type(float)
assert runtime.name == 'Float'
@lambda_solid(input_defs=[InputDefinition('num', int)])
def add_one(num):
return num + 1
assert add_one.input_defs[0].runtime_type.name == 'Int'
runtime = resolve_to_runtime_type(float)
runtime.type_check(1.0)
with pytest.raises(Failure):
runtime.type_check(1)
runtime = resolve_to_runtime_type(bool)
runtime.type_check(True)
with pytest.raises(Failure):
runtime.type_check(1)
def test_runtime_optional_set():
set_runtime_type = resolve_to_runtime_type(
Optional[create_typed_runtime_set(int)])
set_runtime_type.type_check({1})
set_runtime_type.type_check(set())
set_runtime_type.type_check(None)
res = set_runtime_type.type_check('nope')
assert not res.success
res = set_runtime_type.type_check({'nope'})
assert not res.success
