def test_nullable_list_combos_coerciion(): list_of_int = resolve_to_runtime_type(List[Int]) assert_failure(list_of_int, None) assert_success(list_of_int, []) assert_success(list_of_int, [1]) assert_failure(list_of_int, [None]) nullable_int_of_list = resolve_to_runtime_type(Optional[List[Int]]) assert_success(nullable_int_of_list, None) assert_success(nullable_int_of_list, []) assert_success(nullable_int_of_list, [1]) assert_failure(nullable_int_of_list, [None]) list_of_nullable_int = resolve_to_runtime_type(List[Optional[Int]]) assert_failure(list_of_nullable_int, None) assert_success(list_of_nullable_int, []) assert_success(list_of_nullable_int, [1]) assert_success(list_of_nullable_int, [None]) nullable_list_of_nullable_int = resolve_to_runtime_type( Optional[List[Optional[Int]]]) assert_success(nullable_list_of_nullable_int, None) assert_success(nullable_list_of_nullable_int, []) assert_success(nullable_list_of_nullable_int, [1]) assert_success(nullable_list_of_nullable_int, [None])
def test_file_system_intermediate_store_with_composite_type_storage_plugin(): run_id = str(uuid.uuid4()) intermediate_store = build_fs_intermediate_store( DagsterInstance.ephemeral().intermediates_directory, run_id=run_id, type_storage_plugin_registry=TypeStoragePluginRegistry([ (RuntimeString, FancyStringFilesystemTypeStoragePlugin) ]), ) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value(['hello'], context, resolve_to_runtime_type(List[String]), ['obj_name']) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value(['hello'], context, resolve_to_runtime_type( Optional[String]), ['obj_name']) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value( ['hello'], context, resolve_to_runtime_type(List[Optional[String]]), ['obj_name']) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value( ['hello'], context, resolve_to_runtime_type(Optional[List[String]]), ['obj_name'])
def test_display_name(): int_runtime = resolve_to_runtime_type(Int) assert int_runtime.display_name == 'Int' list_int_runtime = resolve_to_runtime_type(List[Int]) assert list_int_runtime.display_name == '[Int]' list_list_int_runtime = resolve_to_runtime_type(List[List[Int]]) assert list_list_int_runtime.display_name == '[[Int]]' list_nullable_int_runtime = resolve_to_runtime_type(List[Optional[Int]]) assert list_nullable_int_runtime.display_name == '[Int?]'
def test_nullable_int_coercion(): int_type = resolve_to_runtime_type(Int) assert_type_check(int_type.type_check(1)) res = int_type.type_check(None) assert not res.success nullable_int_type = resolve_to_runtime_type(Optional[Int]) assert_type_check(nullable_int_type.type_check(1)) assert_type_check(nullable_int_type.type_check(None))
def test_make_dagster_type(): OverwriteNameTuple = as_dagster_type(collections.namedtuple('SomeNamedTuple', 'prop')) runtime_type = resolve_to_runtime_type(OverwriteNameTuple) assert runtime_type.name == 'SomeNamedTuple' assert OverwriteNameTuple(prop='foo').prop == 'foo' OverwriteNameTuple = as_dagster_type( collections.namedtuple('SomeNamedTuple', 'prop'), name='OverwriteName' ) runtime_type = resolve_to_runtime_type(OverwriteNameTuple) assert runtime_type.name == 'OverwriteName' assert OverwriteNameTuple(prop='foo').prop == 'foo'
def test_inner_types(): assert resolve_to_runtime_type(Int).inner_types == [] list_int_runtime = resolve_to_runtime_type(List[Int]) assert inner_type_key_set(list_int_runtime) == set(['Int']) list_list_int_runtime = resolve_to_runtime_type(List[List[Int]]) assert inner_type_key_set(list_list_int_runtime) == set( ['Int', 'List.Int']) list_nullable_int_runtime = resolve_to_runtime_type(List[Optional[Int]]) assert inner_type_key_set(list_nullable_int_runtime) == set( ['Int', 'Optional.Int']) assert not list_nullable_int_runtime.is_scalar
def test_dagster_type_decorator(): @dagster_type(name=None) class Foo(object): pass @dagster_type() class Bar(object): pass @dagster_type class Baaz(object): pass assert resolve_to_runtime_type(Foo).name == 'Foo' assert resolve_to_runtime_type(Bar).name == 'Bar' assert resolve_to_runtime_type(Baaz).name == 'Baaz'
def test_file_system_intermediate_store_composite_types(): run_id = str(uuid.uuid4()) instance = DagsterInstance.ephemeral() intermediate_store = build_fs_intermediate_store( instance.intermediates_directory, run_id=run_id) with yield_empty_pipeline_context(instance=instance, run_id=run_id) as context: intermediate_store.set_object([True, False], context, resolve_to_runtime_type(List[Bool]), ['bool']) assert intermediate_store.has_object(context, ['bool']) assert intermediate_store.get_object( context, resolve_to_runtime_type(List[Bool]), ['bool']).obj == [True, False]
def test_nullable_python_object_type(): nullable_type_bar = resolve_to_runtime_type(Optional[Bar]) assert_type_check(nullable_type_bar.type_check(BarObj())) assert_type_check(nullable_type_bar.type_check(None)) res = nullable_type_bar.type_check('not_a_bar') assert not res.success
def __init__(self, name, dagster_type=None, description=None): '' self._name = check_valid_name(name) self._runtime_type = check.inst(resolve_to_runtime_type(dagster_type), RuntimeType) self._description = check.opt_str_param(description, 'description')
def test_file_system_intermediate_store_composite_types_with_custom_serializer_for_inner_type( ): run_id = str(uuid.uuid4()) instance = DagsterInstance.ephemeral() intermediate_store = build_fs_intermediate_store( instance.intermediates_directory, run_id=run_id) with yield_empty_pipeline_context(run_id=run_id, instance=instance) as context: intermediate_store.set_object(['foo', 'bar'], context, resolve_to_runtime_type( List[LowercaseString]), ['list']) assert intermediate_store.has_object(context, ['list']) assert intermediate_store.get_object( context, resolve_to_runtime_type(List[Bool]), ['list']).obj == ['foo', 'bar']
def test_dagster_type_decorator_name_desc(): @dagster_type(name='DifferentName', description='desc') class Something(object): pass runtime_type = resolve_to_runtime_type(Something) assert runtime_type.name == 'DifferentName' assert runtime_type.description == 'desc'
def remap_python_builtin_for_runtime(ttype): '''This function remaps a python type to a Dagster type, or passes it through if it cannot be remapped. ''' from dagster.core.types.runtime_type import resolve_to_runtime_type check.param_invariant(is_supported_runtime_python_builtin(ttype), 'ttype') return resolve_to_runtime_type(SUPPORTED_RUNTIME_BUILTINS[ttype])
def get_intermediate(self, context, step_key, dagster_type, output_name='result'): return self.get_object( context=context, runtime_type=resolve_to_runtime_type(dagster_type), paths=self.paths_for_intermediate(step_key, output_name), )
def test_gcs_intermediate_store_composite_types_with_custom_serializer_for_inner_type( gcs_bucket): run_id = str(uuid.uuid4()) intermediate_store = GCSIntermediateStore(run_id=run_id, gcs_bucket=gcs_bucket) with yield_empty_pipeline_context(run_id=run_id) as context: try: intermediate_store.set_object( ['foo', 'bar'], context, resolve_to_runtime_type(List[LowercaseString]), ['list'], ) assert intermediate_store.has_object(context, ['list']) assert intermediate_store.get_object( context, resolve_to_runtime_type(List[Bool]), ['list']).obj == ['foo', 'bar'] finally: intermediate_store.rm_object(context, ['foo'])
def test_python_mapping(): runtime = resolve_to_runtime_type(int) assert runtime.name == 'Int' runtime = resolve_to_runtime_type(str) assert runtime.name == 'String' runtime = resolve_to_runtime_type(bool) assert runtime.name == 'Bool' runtime = resolve_to_runtime_type(float) assert runtime.name == 'Float' @lambda_solid(input_defs=[InputDefinition('num', int)]) def add_one(num): return num + 1 assert add_one.input_defs[0].runtime_type.name == 'Int' runtime = resolve_to_runtime_type(float) runtime.type_check(1.0) res = runtime.type_check(1) assert not res.success runtime = resolve_to_runtime_type(bool) runtime.type_check(True) res = runtime.type_check(1) assert not res.success
def test_runtime_optional_set(): set_runtime_type = resolve_to_runtime_type( Optional[create_typed_runtime_set(int)]) set_runtime_type.type_check({1}) set_runtime_type.type_check(set()) set_runtime_type.type_check(None) res = set_runtime_type.type_check('nope') assert not res.success res = set_runtime_type.type_check({'nope'}) assert not res.success
def test_gcs_intermediate_store_with_composite_type_storage_plugin(gcs_bucket): run_id = str(uuid.uuid4()) intermediate_store = GCSIntermediateStore( run_id=run_id, gcs_bucket=gcs_bucket, type_storage_plugin_registry=TypeStoragePluginRegistry([ (RuntimeString, FancyStringGCSTypeStoragePlugin) ]), ) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value(['hello'], context, resolve_to_runtime_type(List[String]), ['obj_name'])
def check_dagster_type(dagster_type, value): '''Test a custom Dagster type. Args: dagster_type (Any): The Dagster type to test. Should be one of the :ref:`built-in types <builtin>`, a dagster type explicitly constructed with :py:func:`as_dagster_type`, :py:func:`@dagster_type <dagster_type>`, or :py:func:`define_python_dagster_type`, or a Python type. value (Any): The runtime value to test. Returns: TypeCheck: The result of the type check. Examples: .. code-block:: python assert check_dagster_type(Dict[Any, Any], {'foo': 'bar'}).success ''' if is_typing_type(dagster_type): raise DagsterInvariantViolationError( ( 'Must pass in a type from dagster module. You passed {dagster_type} ' 'which is part of python\'s typing module.' ).format(dagster_type=dagster_type) ) runtime_type = resolve_to_runtime_type(dagster_type) type_check = runtime_type.type_check(value) if not isinstance(type_check, TypeCheck): raise DagsterInvariantViolationError( 'Type checks can only return TypeCheck. Type {type_name} returned {value}.'.format( type_name=runtime_type.name, value=repr(type_check) ) ) return type_check
def __init__(self, dagster_type=None, name=None, description=None, is_optional=False): self._name = check_valid_name(check.opt_str_param(name, 'name', DEFAULT_OUTPUT)) self._runtime_type = resolve_to_runtime_type(dagster_type) self._description = check.opt_str_param(description, 'description') self._optional = check.bool_param(is_optional, 'is_optional')
def test_comparison(): # Base types assert resolve_to_runtime_type(Any) == resolve_to_runtime_type(Any) assert resolve_to_runtime_type(String) == resolve_to_runtime_type(String) assert resolve_to_runtime_type(Bool) == resolve_to_runtime_type(Bool) assert resolve_to_runtime_type(Float) == resolve_to_runtime_type(Float) assert resolve_to_runtime_type(Int) == resolve_to_runtime_type(Int) assert resolve_to_runtime_type(Path) == resolve_to_runtime_type(Path) assert resolve_to_runtime_type(String) == resolve_to_runtime_type(String) assert resolve_to_runtime_type(Nothing) == resolve_to_runtime_type(Nothing) assert resolve_to_runtime_type( Optional[String]) == resolve_to_runtime_type(Optional[String]) types = [Any, Bool, Float, Int, Path, String, Nothing] non_equal_pairs = [(t1, t2) for t1 in types for t2 in types if t1 != t2] for t1, t2 in non_equal_pairs: assert resolve_to_runtime_type(t1) != resolve_to_runtime_type(t2) assert resolve_to_runtime_type( Optional[String]) != resolve_to_runtime_type(Optional[Int]) # List type assert resolve_to_runtime_type(List) == resolve_to_runtime_type(List) assert resolve_to_runtime_type(List[String]) == resolve_to_runtime_type( List[String]) assert resolve_to_runtime_type(List[List[Int]]) == resolve_to_runtime_type( List[List[Int]]) assert resolve_to_runtime_type( List[Optional[String]]) == resolve_to_runtime_type( List[Optional[String]]) assert resolve_to_runtime_type(List[String]) != resolve_to_runtime_type( List[Int]) assert resolve_to_runtime_type( List[List[String]]) != resolve_to_runtime_type(List[List[Int]]) assert resolve_to_runtime_type(List[String]) != resolve_to_runtime_type( List[Optional[String]]) # Tuple type assert resolve_to_runtime_type(Tuple) == resolve_to_runtime_type(Tuple) assert resolve_to_runtime_type( Tuple[String, Int]) == resolve_to_runtime_type(Tuple[String, Int]) assert resolve_to_runtime_type( Tuple[Tuple[String, Int]]) == resolve_to_runtime_type(Tuple[Tuple[String, Int]]) assert resolve_to_runtime_type(Tuple[Optional[String], Int]) == resolve_to_runtime_type( Tuple[Optional[String], Int]) assert resolve_to_runtime_type( Tuple[String, Int]) != resolve_to_runtime_type(Tuple[Int, String]) assert resolve_to_runtime_type( Tuple[Tuple[String, Int]]) != resolve_to_runtime_type( Tuple[Tuple[Int, String]]) assert resolve_to_runtime_type(Tuple[String]) != resolve_to_runtime_type( Tuple[Optional[String]]) # Set type assert resolve_to_runtime_type(Set) == resolve_to_runtime_type(Set) assert resolve_to_runtime_type(Set[String]) == resolve_to_runtime_type( Set[String]) assert resolve_to_runtime_type(Set[Set[Int]]) == resolve_to_runtime_type( Set[Set[Int]]) assert resolve_to_runtime_type( Set[Optional[String]]) == resolve_to_runtime_type( Set[Optional[String]]) assert resolve_to_runtime_type(Set[String]) != resolve_to_runtime_type( Set[Int]) assert resolve_to_runtime_type( Set[Set[String]]) != resolve_to_runtime_type(Set[Set[Int]]) assert resolve_to_runtime_type(Set[String]) != resolve_to_runtime_type( Set[Optional[String]]) # Dict type assert resolve_to_runtime_type(Dict) == resolve_to_runtime_type(Dict) assert resolve_to_runtime_type( Dict[String, Int]) == resolve_to_runtime_type(Dict[String, Int]) assert resolve_to_runtime_type( Dict[String, Dict[String, Int]]) == resolve_to_runtime_type(Dict[String, Dict[String, Int]]) assert resolve_to_runtime_type( Dict[String, Int]) != resolve_to_runtime_type(Dict[Int, String]) assert resolve_to_runtime_type( Dict[Int, Dict[String, Int]]) != resolve_to_runtime_type( Dict[String, Dict[String, Int]])
def test_make_dagster_type_from_builtin(): OrderedDict = as_dagster_type(collections.OrderedDict) assert OrderedDict is collections.OrderedDict assert OrderedDict([('foo', 'bar')]) == collections.OrderedDict([('foo', 'bar')]) assert isinstance(resolve_to_runtime_type(OrderedDict), RuntimeType) assert resolve_to_runtime_type(OrderedDict).python_type is collections.OrderedDict
def test_is_any(): assert not resolve_to_runtime_type(Int).is_any assert resolve_to_runtime_type(Int).is_scalar