def test_file_system_intermediate_store_with_composite_type_storage_plugin(): run_id = str(uuid.uuid4()) intermediate_store = build_fs_intermediate_store( DagsterInstance.ephemeral().intermediates_directory, run_id=run_id, type_storage_plugin_registry=TypeStoragePluginRegistry( {RuntimeString.inst(): FancyStringFilesystemTypeStoragePlugin}), ) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value(['hello'], context, resolve_to_runtime_type(List[String]), ['obj_name']) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value(['hello'], context, resolve_to_runtime_type( Optional[String]), ['obj_name']) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value( ['hello'], context, resolve_to_runtime_type(List[Optional[String]]), ['obj_name']) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value( ['hello'], context, resolve_to_runtime_type(Optional[List[String]]), ['obj_name'])
def test_file_system_intermediate_store_composite_types_with_custom_serializer_for_inner_type( ): run_id = str(uuid.uuid4()) intermediate_store = FileSystemIntermediateStore(run_id=run_id) assert intermediate_store.root == os.path.join( seven.get_system_temp_directory(), 'dagster', 'runs', run_id, 'files') with yield_empty_pipeline_context(run_id=run_id) as context: try: intermediate_store.set_object( ['foo', 'bar'], context, resolve_to_runtime_type(List(LowercaseString)).inst(), ['list'], ) assert intermediate_store.has_object(context, ['list']) assert intermediate_store.get_object( context, resolve_to_runtime_type(List(Bool)).inst(), ['list']) == ['foo', 'bar'] finally: try: shutil.rmtree(intermediate_store.root) except seven.FileNotFoundError: pass
def test_file_system_intermediate_store_with_composite_type_storage_plugin(): run_id = str(uuid.uuid4()) # FIXME need a dedicated test bucket intermediate_store = FileSystemIntermediateStore( run_id=run_id, type_storage_plugin_registry=TypeStoragePluginRegistry( {RuntimeString.inst(): FancyStringFilesystemTypeStoragePlugin}), ) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value(['hello'], context, resolve_to_runtime_type(List[String]), ['obj_name']) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value(['hello'], context, resolve_to_runtime_type( Optional[String]), ['obj_name']) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value( ['hello'], context, resolve_to_runtime_type(List[Optional[String]]), ['obj_name']) with yield_empty_pipeline_context(run_id=run_id) as context: with pytest.raises(check.NotImplementedCheckError): intermediate_store.set_value( ['hello'], context, resolve_to_runtime_type(Optional[List[String]]), ['obj_name'])
def test_file_system_object_store_composite_types(): run_id = str(uuid.uuid4()) object_store = FileSystemObjectStore(run_id=run_id) assert object_store.root == os.path.join(seven.get_system_temp_directory(), 'dagster', 'runs', run_id, 'files') with yield_pipeline_execution_context(PipelineDefinition([]), {}, RunConfig(run_id=run_id)) as context: try: object_store.set_object([True, False], context, resolve_to_runtime_type( List_(Bool_)).inst(), ['bool']) assert object_store.has_object(context, ['bool']) assert object_store.get_object( context, resolve_to_runtime_type(List_(Bool_)).inst(), ['bool']) == [True, False] finally: try: shutil.rmtree(object_store.root) except seven.FileNotFoundError: pass
def test_file_system_object_store_composite_types_with_custom_serializer_for_inner_type( ): run_id = str(uuid.uuid4()) object_store = FileSystemObjectStore(run_id=run_id) assert object_store.root == os.path.join(seven.get_system_temp_directory(), 'dagster', 'runs', run_id, 'files') with yield_pipeline_execution_context(PipelineDefinition([]), {}, RunConfig(run_id=run_id)) as context: try: object_store.set_object( ['foo', 'bar'], context, resolve_to_runtime_type(List_(LowercaseString)).inst(), ['list'], ) assert object_store.has_object(context, ['list']) assert object_store.get_object( context, resolve_to_runtime_type(List_(Bool_)).inst(), ['list']) == ['foo', 'bar'] finally: try: shutil.rmtree(object_store.root) except seven.FileNotFoundError: pass
def test_nullable_list_combos_coerciion(): list_of_int = resolve_to_runtime_type(List(Int)) assert_failure(list_of_int.coerce_runtime_value, None) assert_success(list_of_int.coerce_runtime_value, []) assert_success(list_of_int.coerce_runtime_value, [1]) assert_failure(list_of_int.coerce_runtime_value, [None]) nullable_int_of_list = resolve_to_runtime_type(Nullable(List(Int))) assert_success(nullable_int_of_list.coerce_runtime_value, None) assert_success(nullable_int_of_list.coerce_runtime_value, []) assert_success(nullable_int_of_list.coerce_runtime_value, [1]) assert_failure(nullable_int_of_list.coerce_runtime_value, [None]) list_of_nullable_int = resolve_to_runtime_type(List(Nullable(Int))) assert_failure(list_of_nullable_int.coerce_runtime_value, None) assert_success(list_of_nullable_int.coerce_runtime_value, []) assert_success(list_of_nullable_int.coerce_runtime_value, [1]) assert_success(list_of_nullable_int.coerce_runtime_value, [None]) nullable_list_of_nullable_int = resolve_to_runtime_type(Nullable(List(Nullable(Int)))) assert_success(nullable_list_of_nullable_int.coerce_runtime_value, None) assert_success(nullable_list_of_nullable_int.coerce_runtime_value, []) assert_success(nullable_list_of_nullable_int.coerce_runtime_value, [1]) assert_success(nullable_list_of_nullable_int.coerce_runtime_value, [None])
def test_nullable_list_combos_coerciion(): list_of_int = resolve_to_runtime_type(List[Int]) assert_failure(list_of_int, None) assert_success(list_of_int, []) assert_success(list_of_int, [1]) assert_failure(list_of_int, [None]) nullable_int_of_list = resolve_to_runtime_type(Optional[List[Int]]) assert_success(nullable_int_of_list, None) assert_success(nullable_int_of_list, []) assert_success(nullable_int_of_list, [1]) assert_failure(nullable_int_of_list, [None]) list_of_nullable_int = resolve_to_runtime_type(List[Optional[Int]]) assert_failure(list_of_nullable_int, None) assert_success(list_of_nullable_int, []) assert_success(list_of_nullable_int, [1]) assert_success(list_of_nullable_int, [None]) nullable_list_of_nullable_int = resolve_to_runtime_type( Optional[List[Optional[Int]]]) assert_success(nullable_list_of_nullable_int, None) assert_success(nullable_list_of_nullable_int, []) assert_success(nullable_list_of_nullable_int, [1]) assert_success(nullable_list_of_nullable_int, [None])
def test_make_dagster_type_from_builtin(): OrderedDict = as_dagster_type(collections.OrderedDict) assert OrderedDict is collections.OrderedDict assert OrderedDict([('foo', 'bar')]) == collections.OrderedDict([('foo', 'bar')]) assert isinstance(resolve_to_runtime_type(OrderedDict), RuntimeType) assert resolve_to_runtime_type( OrderedDict).python_type is collections.OrderedDict
def test_nullable_int_coercion(): int_type = resolve_to_runtime_type(Int) assert_type_check(int_type.type_check(1)) with pytest.raises(Failure): int_type.type_check(None) nullable_int_type = resolve_to_runtime_type(Optional[Int]) assert_type_check(nullable_int_type.type_check(1)) assert_type_check(nullable_int_type.type_check(None))
def test_display_name(): int_runtime = resolve_to_runtime_type(Int) assert int_runtime.display_name == 'Int' list_int_runtime = resolve_to_runtime_type(List[Int]) assert list_int_runtime.display_name == '[Int]' list_list_int_runtime = resolve_to_runtime_type(List[List[Int]]) assert list_list_int_runtime.display_name == '[[Int]]' list_nullable_int_runtime = resolve_to_runtime_type(List[Optional[Int]]) assert list_nullable_int_runtime.display_name == '[Int?]'
def test_nullable_int_coercion(): int_type = resolve_to_runtime_type(Int) assert int_type.coerce_runtime_value(1) == 1 with pytest.raises(DagsterRuntimeCoercionError): assert int_type.coerce_runtime_value(None) nullable_int_type = resolve_to_runtime_type(Nullable(Int)) assert nullable_int_type.coerce_runtime_value(1) == 1 assert nullable_int_type.coerce_runtime_value(None) is None
def test_display_name(): int_runtime = resolve_to_runtime_type(Int) assert int_runtime.display_name == 'Int' list_int_runtime = resolve_to_runtime_type(List(Int)) assert list_int_runtime.display_name == '[Int]' list_list_int_runtime = resolve_to_runtime_type(List(List(Int))) assert list_list_int_runtime.display_name == '[[Int]]' list_nullable_int_runtime = resolve_to_runtime_type(List(Nullable(Int))) assert list_nullable_int_runtime.display_name == '[Int?]'
def test_inner_types(): assert resolve_to_runtime_type(Int).inner_types == [] list_int_runtime = resolve_to_runtime_type(List[Int]) assert inner_type_key_set(list_int_runtime) == set(['Int']) list_list_int_runtime = resolve_to_runtime_type(List[List[Int]]) assert inner_type_key_set(list_list_int_runtime) == set(['Int', 'List.Int']) list_nullable_int_runtime = resolve_to_runtime_type(List[Optional[Int]]) assert inner_type_key_set(list_nullable_int_runtime) == set(['Int', 'Optional.Int'])
def test_make_dagster_type(): OverwriteNameTuple = as_dagster_type( collections.namedtuple('SomeNamedTuple', 'prop')) runtime_type = resolve_to_runtime_type(OverwriteNameTuple) assert runtime_type.name == 'SomeNamedTuple' assert OverwriteNameTuple(prop='foo').prop == 'foo' OverwriteNameTuple = as_dagster_type(collections.namedtuple( 'SomeNamedTuple', 'prop'), name='OverwriteName') runtime_type = resolve_to_runtime_type(OverwriteNameTuple) assert runtime_type.name == 'OverwriteName' assert OverwriteNameTuple(prop='foo').prop == 'foo'
def test_inner_types(): assert resolve_to_runtime_type(Int).inner_types == [] list_int_runtime = resolve_to_runtime_type(List(Int)) assert inner_type_key_set(list_int_runtime) == set(['Int']) list_list_int_runtime = resolve_to_runtime_type(List(List(Int))) assert inner_type_key_set(list_list_int_runtime) == set( ['Int', 'List.Int']) list_nullable_int_runtime = resolve_to_runtime_type(List(Nullable(Int))) assert inner_type_key_set(list_nullable_int_runtime) == set( ['Int', 'Nullable.Int'])
def test_python_mapping(): int_runtime = resolve_to_runtime_type(int) assert int_runtime.name == 'Int' int_runtime = resolve_to_runtime_type(str) assert int_runtime.name == 'String' int_runtime = resolve_to_runtime_type(bool) assert int_runtime.name == 'Bool' int_runtime = resolve_to_runtime_type(float) assert int_runtime.name == 'Float' @lambda_solid(inputs=[InputDefinition('num', int)]) def add_one(num): return num + 1 assert add_one.input_defs[0].runtime_type.name == 'Int'
def __init__(self, name, dagster_type=None, description=None): '' self.name = check_valid_name(name) self.runtime_type = check.inst(resolve_to_runtime_type(dagster_type), RuntimeType) self.description = check.opt_str_param(description, 'description')
def check_dagster_type(dagster_type, value): '''Test a custom Dagster type. Args: dagster_type (Any): The Dagster type to test. Should be one of the :ref:`built-in types <builtin>`, a dagster type explicitly constructed with :py:func:`as_dagster_type`, :py:func:`@dagster_type <dagster_type>`, or :py:func:`define_python_dagster_type`, or a Python type. value (Any): The runtime value to test. Returns: TypeCheck: The result of the type check. **Examples**: .. code-block:: python assert check_dagster_type(Dict[Any, Any], {'foo': 'bar'}).success ''' runtime_type = resolve_to_runtime_type(dagster_type) type_check = runtime_type.type_check(value) if not isinstance(type_check, TypeCheck): raise DagsterInvariantViolationError( ('Type checks can only return TypeCheck. Type ' '{type_name} returned {value}.').format( type_name=runtime_type.name, value=repr(type_check))) return type_check
def test_dagster_type_decorator(): @dagster_type(name=None) class Foo(object): pass @dagster_type() class Bar(object): pass @dagster_type class Baaz(object): pass assert resolve_to_runtime_type(Foo).name == 'Foo' assert resolve_to_runtime_type(Bar).name == 'Bar' assert resolve_to_runtime_type(Baaz).name == 'Baaz'
def test_nullable_python_object_type(): nullable_type_bar = resolve_to_runtime_type(Optional[Bar]) assert_type_check(nullable_type_bar.type_check(BarObj())) assert_type_check(nullable_type_bar.type_check(None)) with pytest.raises(Failure): nullable_type_bar.type_check('not_a_bar')
def test_dagster_type_decorator_name_desc(): @dagster_type(name='DifferentName', description='desc') class Something(object): pass runtime_type = resolve_to_runtime_type(Something) assert runtime_type.name == 'DifferentName' assert runtime_type.description == 'desc'
def test_file_system_intermediate_store_composite_types(): run_id = str(uuid.uuid4()) instance = DagsterInstance.ephemeral() intermediate_store = FilesystemIntermediateStore.for_instance( instance=instance, run_id=run_id) with yield_empty_pipeline_context(instance=instance, run_id=run_id) as context: intermediate_store.set_object([True, False], context, resolve_to_runtime_type( List[Bool]).inst(), ['bool']) assert intermediate_store.has_object(context, ['bool']) assert intermediate_store.get_object( context, resolve_to_runtime_type(List[Bool]).inst(), ['bool']).obj == [True, False]
def test_nullable_python_object_type(): nullable_type_bar = resolve_to_runtime_type(Nullable(Bar)) assert nullable_type_bar.coerce_runtime_value(BarObj()) assert nullable_type_bar.coerce_runtime_value(None) is None with pytest.raises(DagsterRuntimeCoercionError): nullable_type_bar.coerce_runtime_value('not_a_bar')
def test_file_system_intermediate_store_composite_types_with_custom_serializer_for_inner_type( ): run_id = str(uuid.uuid4()) instance = DagsterInstance.ephemeral() intermediate_store = FilesystemIntermediateStore.for_instance( instance, run_id=run_id) with yield_empty_pipeline_context(run_id=run_id, instance=instance) as context: intermediate_store.set_object( ['foo', 'bar'], context, resolve_to_runtime_type(List[LowercaseString]).inst(), ['list']) assert intermediate_store.has_object(context, ['list']) assert intermediate_store.get_object( context, resolve_to_runtime_type(List[Bool]).inst(), ['list']).obj == ['foo', 'bar']
def __init__(self, name, dagster_type=None, expectations=None, description=None): '' self.name = check_valid_name(name) self.runtime_type = check.inst(resolve_to_runtime_type(dagster_type), RuntimeType) self.expectations = check.opt_list_param( expectations, 'expectations', of_type=IOExpectationDefinition ) self.description = check.opt_str_param(description, 'description')
def get_intermediate(self, context, step_key, dagster_type, output_name='result'): return self.get_object( context=context, runtime_type=resolve_to_runtime_type(dagster_type), paths=self.paths_for_intermediate(step_key, output_name), )
def test_s3_intermediate_store_composite_types_with_custom_serializer_for_inner_type(s3_bucket): run_id = str(uuid.uuid4()) intermediate_store = S3IntermediateStore(run_id=run_id, s3_bucket=s3_bucket) with yield_empty_pipeline_context(run_id=run_id) as context: try: intermediate_store.set_object( ['foo', 'bar'], context, resolve_to_runtime_type(List[LowercaseString]).inst(), ['list'], ) assert intermediate_store.has_object(context, ['list']) assert intermediate_store.get_object( context, resolve_to_runtime_type(List[Bool]).inst(), ['list'] ).obj == ['foo', 'bar'] finally: intermediate_store.rm_object(context, ['foo'])
def __init__(self, dagster_type=None, name=None, description=None, is_optional=False): self._name = check_valid_name( check.opt_str_param(name, 'name', DEFAULT_OUTPUT)) self._runtime_type = resolve_to_runtime_type(dagster_type) self._description = check.opt_str_param(description, 'description') self._optional = check.bool_param(is_optional, 'is_optional')
def test_python_mapping(): runtime = resolve_to_runtime_type(int) assert runtime.name == 'Int' runtime = resolve_to_runtime_type(str) assert runtime.name == 'String' runtime = resolve_to_runtime_type(bool) assert runtime.name == 'Bool' runtime = resolve_to_runtime_type(float) assert runtime.name == 'Float' @lambda_solid(input_defs=[InputDefinition('num', int)]) def add_one(num): return num + 1 assert add_one.input_defs[0].runtime_type.name == 'Int' runtime = resolve_to_runtime_type(float) runtime.type_check(1.0) with pytest.raises(Failure): runtime.type_check(1) runtime = resolve_to_runtime_type(bool) runtime.type_check(True) with pytest.raises(Failure): runtime.type_check(1)
def test_runtime_optional_set(): set_runtime_type = resolve_to_runtime_type( Optional[create_typed_runtime_set(int)]) set_runtime_type.type_check({1}) set_runtime_type.type_check(set()) set_runtime_type.type_check(None) res = set_runtime_type.type_check('nope') assert not res.success res = set_runtime_type.type_check({'nope'}) assert not res.success