def define_test_type_pipeline():
return PipelineDefinition(
name='test_type_pipeline',
solids=[
define_solid_for_test_type('int_config', Int),
define_solid_for_test_type('list_of_int_config', List(Int)),
define_solid_for_test_type('nullable_list_of_int_config',
Nullable(List(Int))),
define_solid_for_test_type('list_of_nullable_int_config',
List(Nullable(Int))),
define_solid_for_test_type('nullable_list_of_nullable_int_config',
Nullable(List(Nullable(Int)))),
define_solid_for_test_type(
'simple_dict',
Dict({
'int_field': Field(Int),
'string_field': Field(String)
})),
define_solid_for_test_type(
'dict_with_optional_field',
Dict({
'nullable_int_field': Field(Nullable(Int)),
'optional_int_field': Field(Int, is_optional=True),
'string_list_field': Field(List(String)),
}),
),
define_solid_for_test_type(
'nested_dict',
Dict({'nested': Field(Dict({'int_field': Field(Int)}))})),
],
)
def test_nullable_dict():
dict_with_int = Dict({'int_field': Field(Int)})
assert not eval_config_value_from_dagster_type(dict_with_int, None).success
assert not eval_config_value_from_dagster_type(dict_with_int, {}).success
assert not eval_config_value_from_dagster_type(dict_with_int, {'int_field': None}).success
assert eval_config_value_from_dagster_type(dict_with_int, {'int_field': 1}).success
nullable_dict_with_int = Nullable(Dict({'int_field': Field(Int)}))
assert eval_config_value_from_dagster_type(nullable_dict_with_int, None).success
assert not eval_config_value_from_dagster_type(nullable_dict_with_int, {}).success
assert not eval_config_value_from_dagster_type(
nullable_dict_with_int, {'int_field': None}
).success
assert eval_config_value_from_dagster_type(nullable_dict_with_int, {'int_field': 1}).success
dict_with_nullable_int = Dict({'int_field': Field(Nullable(Int))})
assert not eval_config_value_from_dagster_type(dict_with_nullable_int, None).success
assert not eval_config_value_from_dagster_type(dict_with_nullable_int, {}).success
assert eval_config_value_from_dagster_type(dict_with_nullable_int, {'int_field': None}).success
assert eval_config_value_from_dagster_type(dict_with_nullable_int, {'int_field': 1}).success
nullable_dict_with_nullable_int = Nullable(Dict({'int_field': Field(Nullable(Int))}))
assert eval_config_value_from_dagster_type(nullable_dict_with_nullable_int, None).success
assert not eval_config_value_from_dagster_type(nullable_dict_with_nullable_int, {}).success
assert eval_config_value_from_dagster_type(
nullable_dict_with_nullable_int, {'int_field': None}
).success
assert eval_config_value_from_dagster_type(
nullable_dict_with_nullable_int, {'int_field': 1}
).success
def test_nullable_list():
list_of_ints = List(Int)
assert not eval_config_value_from_dagster_type(list_of_ints, None).success
assert eval_config_value_from_dagster_type(list_of_ints, []).success
assert not eval_config_value_from_dagster_type(list_of_ints, [None]).success
assert eval_config_value_from_dagster_type(list_of_ints, [1]).success
nullable_list_of_ints = Nullable(List(Int))
assert eval_config_value_from_dagster_type(nullable_list_of_ints, None).success
assert eval_config_value_from_dagster_type(nullable_list_of_ints, []).success
assert not eval_config_value_from_dagster_type(nullable_list_of_ints, [None]).success
assert eval_config_value_from_dagster_type(nullable_list_of_ints, [1]).success
list_of_nullable_ints = List(Nullable(Int))
assert not eval_config_value_from_dagster_type(list_of_nullable_ints, None).success
assert eval_config_value_from_dagster_type(list_of_nullable_ints, []).success
assert eval_config_value_from_dagster_type(list_of_nullable_ints, [None]).success
assert eval_config_value_from_dagster_type(list_of_nullable_ints, [1]).success
nullable_list_of_nullable_ints = Nullable(List(Nullable(Int)))
assert eval_config_value_from_dagster_type(nullable_list_of_nullable_ints, None).success
assert eval_config_value_from_dagster_type(nullable_list_of_nullable_ints, []).success
assert eval_config_value_from_dagster_type(nullable_list_of_nullable_ints, [None]).success
assert eval_config_value_from_dagster_type(nullable_list_of_nullable_ints, [1]).success
def test_nullable_int():
assert not eval_config_value_from_dagster_type(Int, None).success
assert eval_config_value_from_dagster_type(Int, 0).success
assert eval_config_value_from_dagster_type(Int, 1).success
assert eval_config_value_from_dagster_type(Nullable(Int), None).success
assert eval_config_value_from_dagster_type(Nullable(Int), 0).success
assert eval_config_value_from_dagster_type(Nullable(Int), 1).success
def define_more_complicated_nested_config():
return PipelineDefinition(
name='more_complicated_nested_config',
solids=[
SolidDefinition(
name='a_solid_with_multilayered_config',
inputs=[],
outputs=[],
transform_fn=lambda *_args: None,
config_field=Field(
Dict(
{
'field_one': Field(String),
'field_two': Field(String, is_optional=True),
'field_three': Field(
String, is_optional=True, default_value='some_value'
),
'nested_field': Field(
Dict(
{
'field_four_str': Field(String),
'field_five_int': Field(Int),
'field_six_nullable_int_list': Field(
List(Nullable(Int)), is_optional=True
),
}
)
),
}
)
),
)
],
)
def test_display_name():
int_runtime = resolve_to_runtime_type(Int)
assert int_runtime.display_name == 'Int'
list_int_runtime = resolve_to_runtime_type(List(Int))
assert list_int_runtime.display_name == '[Int]'
list_list_int_runtime = resolve_to_runtime_type(List(List(Int)))
assert list_list_int_runtime.display_name == '[[Int]]'
list_nullable_int_runtime = resolve_to_runtime_type(List(Nullable(Int)))
assert list_nullable_int_runtime.display_name == '[Int?]'
def test_inner_types():
assert resolve_to_runtime_type(Int).inner_types == []
list_int_runtime = resolve_to_runtime_type(List(Int))
assert inner_type_key_set(list_int_runtime) == set(['Int'])
list_list_int_runtime = resolve_to_runtime_type(List(List(Int)))
assert inner_type_key_set(list_list_int_runtime) == set(
['Int', 'List.Int'])
list_nullable_int_runtime = resolve_to_runtime_type(List(Nullable(Int)))
assert inner_type_key_set(list_nullable_int_runtime) == set(
['Int', 'Nullable.Int'])
def test_wrapping_nothing():
with pytest.raises(DagsterInvalidDefinitionError):
@lambda_solid(output=OutputDefinition(List(Nothing)))
def _():
pass
with pytest.raises(DagsterInvalidDefinitionError):
@lambda_solid(inputs=[InputDefinition(List(Nothing))])
def _():
pass
with pytest.raises(DagsterInvalidDefinitionError):
@lambda_solid(output=OutputDefinition(Nullable(Nothing)))
def _():
pass
with pytest.raises(DagsterInvalidDefinitionError):
@lambda_solid(inputs=[InputDefinition(Nullable(Nothing))])
def _():
pass
def test_file_system_intermediate_store_with_composite_type_storage_plugin():
run_id = str(uuid.uuid4())
# FIXME need a dedicated test bucket
intermediate_store = FileSystemIntermediateStore(
run_id=run_id,
types_to_register={
RuntimeString.inst(): FancyStringFilesystemTypeStoragePlugin
},
)
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(['hello'], context,
resolve_to_runtime_type(List(String)),
['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(['hello'], context,
resolve_to_runtime_type(
Nullable(String)), ['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(['hello'], context,
resolve_to_runtime_type(
List(Nullable(String))),
['obj_name'])
with yield_empty_pipeline_context(run_id=run_id) as context:
with pytest.raises(check.NotImplementedCheckError):
intermediate_store.set_value(['hello'], context,
resolve_to_runtime_type(
Nullable(List(String))),
['obj_name'])
def test_single_level_dict_lists_and_nullable():
output = print_type_to_string(
Dict({
'nullable_int_field': Field(Nullable(Int)),
'optional_int_field': Field(Int, is_optional=True),
'string_list_field': Field(List(String)),
}))
expected = '''{
nullable_int_field: Int?
optional_int_field?: Int
string_list_field: [String]
}'''
assert output == expected
def test_multilevel_good_error_handling_config_solids_name_solids():
@solid(config_field=Field(Nullable(Int)))
def good_error_handling(_context):
pass
pipeline_def = PipelineDefinition(name='multilevel_good_error_handling',
solids=[good_error_handling])
execute_pipeline(
pipeline_def,
environment_dict={'solids': {
'good_error_handling': {
'config': None
}
}})
def define_test_all_scalars_pipeline():
@lambda_solid(inputs=[InputDefinition('num', Int)])
def take_int(num):
return num
@lambda_solid(output=OutputDefinition(Int))
def produce_int():
return 2
@lambda_solid(inputs=[InputDefinition('string', String)])
def take_string(string):
return string
@lambda_solid(output=OutputDefinition(String))
def produce_string():
return 'foo'
@lambda_solid(inputs=[InputDefinition('path', Path)])
def take_path(path):
return path
@lambda_solid(output=OutputDefinition(Path))
def produce_path():
return '/path/to/foo'
@lambda_solid(inputs=[InputDefinition('float_number', Float)])
def take_float(float_number):
return float_number
@lambda_solid(output=OutputDefinition(Float))
def produce_float():
return 3.14
@lambda_solid(inputs=[InputDefinition('bool_value', Bool)])
def take_bool(bool_value):
return bool_value
@lambda_solid(output=OutputDefinition(Bool))
def produce_bool():
return True
@lambda_solid(inputs=[InputDefinition('any_value', Any)])
def take_any(any_value):
return any_value
@lambda_solid(output=OutputDefinition(Any))
def produce_any():
return True
@lambda_solid(inputs=[InputDefinition('string_list', List(String))])
def take_string_list(string_list):
return string_list
@lambda_solid(
inputs=[InputDefinition('nullable_string', Nullable(String))])
def take_nullable_string(nullable_string):
return nullable_string
return PipelineDefinition(
name='test_all_scalars_pipeline',
solids=[
produce_any,
produce_bool,
produce_float,
produce_int,
produce_path,
produce_string,
take_any,
take_bool,
take_float,
take_int,
take_nullable_string,
take_path,
take_string,
take_string_list,
],
)
def test_nullable_list_combos():
assert print_type_to_string(List(Int)) == '[Int]'
assert print_type_to_string(Nullable(List(Int))) == '[Int]?'
assert print_type_to_string(List(Nullable(Int))) == '[Int?]'
assert print_type_to_string(Nullable(List(Nullable(Int)))) == '[Int?]?'
def test_basic_nullable_type_print():
assert print_type_to_string(Nullable(Int)) == 'Int?'
nullable_int = Nullable(Int)
assert_inner_types(nullable_int, Int)