def test_check_solid_list_list_config_correct():
@solid(config_schema={"list_list_int": [[{"bar": int}]]})
def solid_with_config(_):
pass
@pipeline
def single_solid_pipeline():
solid_with_config()
solid_config_key = solid_with_config.config_schema.config_type.key
config_snaps = build_config_schema_snapshot(
single_solid_pipeline).all_config_snaps_by_key
assert solid_config_key in config_snaps
solid_config_snap = config_snaps[solid_config_key]
assert solid_config_snap.kind == ConfigTypeKind.STRICT_SHAPE
assert len(solid_config_snap.fields) == 1
list_list_field = solid_config_snap.fields[0]
list_list_type_key = list_list_field.type_key
assert list_list_type_key.startswith("Array.Array.")
list_list_type = config_snaps[list_list_type_key]
assert list_list_type.kind == ConfigTypeKind.ARRAY
list_snap = config_snaps[list_list_type.inner_type_key]
assert list_snap.kind == ConfigTypeKind.ARRAY
assert config_snaps[
list_snap.inner_type_key].kind == ConfigTypeKind.STRICT_SHAPE
def test_multiple_modes():
@solid
def noop_solid(_):
pass
@resource(config_schema={"a": int})
def a_resource(_):
pass
@resource(config_schema={"b": int})
def b_resource(_):
pass
@pipeline(mode_defs=[
ModeDefinition(name="mode_a", resource_defs={"resource": a_resource}),
ModeDefinition(name="mode_b", resource_defs={"resource": b_resource}),
])
def modez():
noop_solid()
config_snaps = build_config_schema_snapshot(modez).all_config_snaps_by_key
assert a_resource.config_schema.config_type.key in config_snaps
assert b_resource.config_schema.config_type.key in config_snaps
assert get_config_snap(modez, a_resource.config_schema.config_type.key)
assert get_config_snap(modez, b_resource.config_schema.config_type.key)
def test_check_solid_config_correct():
@solid(config_schema={"foo": str})
def solid_with_config(_):
pass
@pipeline
def single_solid_pipeline():
solid_with_config()
solid_config_key = solid_with_config.config_schema.config_type.key
config_snaps = build_config_schema_snapshot(
single_solid_pipeline).all_config_snaps_by_key
assert solid_config_key in config_snaps
solid_config_snap = config_snaps[solid_config_key]
assert solid_config_snap.kind == ConfigTypeKind.STRICT_SHAPE
assert len(solid_config_snap.fields) == 1
foo_field = solid_config_snap.fields[0]
assert foo_field.name == "foo"
assert foo_field.type_key == "String"
def test_kitchen_sink_break_out():
@solid(config_schema=[{
"opt_list_of_int": Field([int], is_required=False),
"nested_dict": {
"list_list": [[int]],
"nested_selector":
Selector({
"some_field": int,
"noneable_list": Noneable([bool])
}),
},
"map": {
str: {
"map_a": int,
"map_b": [str]
},
},
}])
def solid_with_kitchen_sink_config(_):
pass
@pipeline
def single_solid_pipeline():
solid_with_kitchen_sink_config()
config_snaps = build_config_schema_snapshot(
single_solid_pipeline).all_config_snaps_by_key
solid_config_key = solid_with_kitchen_sink_config.config_schema.config_type.key
assert solid_config_key in config_snaps
solid_config_snap = config_snaps[solid_config_key]
assert solid_config_snap.kind == ConfigTypeKind.ARRAY
dict_within_list = config_snaps[solid_config_snap.inner_type_key]
assert len(dict_within_list.fields) == 3
opt_field = dict_within_list.get_field("opt_list_of_int")
assert opt_field.is_required is False
assert config_snaps[opt_field.type_key].kind == ConfigTypeKind.ARRAY
nested_dict = config_snaps[dict_within_list.get_field(
"nested_dict").type_key]
assert len(nested_dict.fields) == 2
nested_selector = config_snaps[nested_dict.get_field(
"nested_selector").type_key]
noneable_list_bool = config_snaps[nested_selector.get_field(
"noneable_list").type_key]
assert noneable_list_bool.kind == ConfigTypeKind.NONEABLE
list_bool = config_snaps[noneable_list_bool.inner_type_key]
assert list_bool.kind == ConfigTypeKind.ARRAY
map = config_snaps[dict_within_list.get_field("map").type_key]
assert map.kind == ConfigTypeKind.MAP
map_dict = config_snaps[map.inner_type_key]
assert len(map_dict.fields) == 2
map_a = config_snaps[map_dict.get_field("map_a").type_key]
assert map_a.kind == ConfigTypeKind.SCALAR
def test_multiple_modes():
@solid
def noop_solid(_):
pass
@resource(config={'a': int})
def a_resource(_):
pass
@resource(config={'b': int})
def b_resource(_):
pass
@pipeline(mode_defs=[
ModeDefinition(name='mode_a', resource_defs={'resource': a_resource}),
ModeDefinition(name='mode_b', resource_defs={'resource': b_resource}),
])
def modez():
noop_solid()
config_snaps = build_config_schema_snapshot(modez).all_config_snaps_by_key
assert a_resource.config_field.config_type.key in config_snaps
assert b_resource.config_field.config_type.key in config_snaps
assert get_config_snap(modez, a_resource.config_field.config_type.key)
assert get_config_snap(modez, b_resource.config_field.config_type.key)
def test_simple_pipeline_smoke_test():
@solid
def solid_without_config(_):
pass
@pipeline
def single_solid_pipeline():
solid_without_config()
config_schema_snapshot = build_config_schema_snapshot(single_solid_pipeline)
assert config_schema_snapshot.all_config_snaps_by_key
serialized = serialize_dagster_namedtuple(config_schema_snapshot)
rehydrated_config_schema_snapshot = deserialize_json_to_dagster_namedtuple(serialized)
assert config_schema_snapshot == rehydrated_config_schema_snapshot
def test_kitchen_sink_break_out():
@solid(config_schema=[{
'opt_list_of_int': Field([int], is_required=False),
'nested_dict': {
'list_list': [[int]],
'nested_selector':
Selector({
'some_field': int,
'noneable_list': Noneable([bool])
}),
},
}])
def solid_with_kitchen_sink_config(_):
pass
@pipeline
def single_solid_pipeline():
solid_with_kitchen_sink_config()
config_snaps = build_config_schema_snapshot(
single_solid_pipeline).all_config_snaps_by_key
solid_config_key = solid_with_kitchen_sink_config.config_schema.config_type.key
assert solid_config_key in config_snaps
solid_config_snap = config_snaps[solid_config_key]
assert solid_config_snap.kind == ConfigTypeKind.ARRAY
dict_within_list = config_snaps[solid_config_snap.inner_type_key]
assert len(dict_within_list.fields) == 2
opt_field = dict_within_list.get_field('opt_list_of_int')
assert opt_field.is_required is False
assert config_snaps[opt_field.type_key].kind == ConfigTypeKind.ARRAY
nested_dict = config_snaps[dict_within_list.get_field(
'nested_dict').type_key]
assert len(nested_dict.fields) == 2
nested_selector = config_snaps[nested_dict.get_field(
'nested_selector').type_key]
noneable_list_bool = config_snaps[nested_selector.get_field(
'noneable_list').type_key]
assert noneable_list_bool.kind == ConfigTypeKind.NONEABLE
list_bool = config_snaps[noneable_list_bool.inner_type_key]
assert list_bool.kind == ConfigTypeKind.ARRAY
def test_scalar_union():
# Requiring resolve calls is bad: https://github.com/dagster-io/dagster/issues/2266
@solid(config_schema=ScalarUnion(resolve_to_config_type(str),
resolve_to_config_type({"bar": str})))
def solid_with_config(_):
pass
@pipeline
def single_solid_pipeline():
solid_with_config()
config_snaps = build_config_schema_snapshot(
single_solid_pipeline).all_config_snaps_by_key
scalar_union_key = solid_with_config.config_schema.config_type.key
assert scalar_union_key in config_snaps
assert config_snaps[
config_snaps[scalar_union_key].scalar_type_key].key == "String"
assert (config_snaps[config_snaps[scalar_union_key].non_scalar_type_key].
kind == ConfigTypeKind.STRICT_SHAPE)
