def get_outputs_field(
solid: Node,
resource_defs: Dict[str, ResourceDefinition],
) -> Optional[Field]:
# if any outputs have configurable output managers, use those for the schema and ignore all type
# materializers
output_manager_fields = {}
for name, output_def in solid.definition.output_dict.items():
output_manager_output_field = get_output_manager_output_field(
solid, output_def, resource_defs)
if output_manager_output_field:
output_manager_fields[name] = output_manager_output_field
if output_manager_fields:
return Field(Shape(output_manager_fields))
# otherwise, use any type materializers for the schema
type_materializer_fields = {}
for name, output_def in solid.definition.output_dict.items():
type_output_field = get_type_output_field(output_def)
if type_output_field:
type_materializer_fields[name] = type_output_field
if type_materializer_fields:
return Field(Array(Shape(type_materializer_fields)), is_required=False)
return None
def get_outputs_field(solid, handle, resource_defs):
check.inst_param(solid, "solid", Solid)
check.inst_param(handle, "handle", SolidHandle)
check.dict_param(resource_defs,
"resource_defs",
key_type=str,
value_type=ResourceDefinition)
# if any outputs have configurable output managers, use those for the schema and ignore all type
# materializers
output_manager_fields = {}
for name, output_def in solid.definition.output_dict.items():
output_manager_output_field = get_output_manager_output_field(
solid, output_def, resource_defs)
if output_manager_output_field:
output_manager_fields[name] = output_manager_output_field
if output_manager_fields:
return Field(Shape(output_manager_fields))
# otherwise, use any type materializers for the schema
type_materializer_fields = {}
for name, output_def in solid.definition.output_dict.items():
type_output_field = get_type_output_field(output_def)
if type_output_field:
type_materializer_fields[name] = type_output_field
if type_materializer_fields:
return Field(Array(Shape(type_materializer_fields)), is_required=False)
return None
def config_type(cls):
return {
"max_concurrent_runs":
Field(config=IntSource, is_required=False),
"tag_concurrency_limits":
Field(
config=Noneable(
Array(
Shape({
"key":
String,
"value":
Field(
ScalarUnion(
scalar_type=String,
non_scalar_schema=Shape(
{"applyLimitPerUniqueValue": Bool}),
),
is_required=False,
),
"limit":
Field(int),
}))),
is_required=False,
),
"dequeue_interval_seconds":
Field(config=IntSource, is_required=False),
}
def __init__(self, inner_dagster_type):
self._inner_dagster_type = check.inst_param(inner_dagster_type,
'inner_dagster_type',
DagsterType)
check.param_invariant(inner_dagster_type.input_hydration_config,
'inner_dagster_type')
self._schema_type = Array(
inner_dagster_type.input_hydration_config.schema_type)
def _construct_array_from_snap(config_type_snap, config_snap_map):
check.list_param(config_type_snap.type_param_keys, 'type_param_keys', str)
check.invariant(
len(config_type_snap.type_param_keys) == 1,
'Expect ARRAY to provide a single inner type. Snapshot provided: {}'.
format(config_type_snap.type_param_keys),
)
return Array(inner_type=construct_config_type_from_snap(
config_snap_map[config_type_snap.type_param_keys[0]], config_snap_map))
def test_multi_type_config_array_dict_fields(dict_config_type, snapshot):
@solid(config=Array(dict_config_type({'foo': Field(int), 'bar': Field(str)})))
def fancy_solid(_):
pass
@pipeline
def noop_pipeline():
fancy_solid()
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline)
solid_def_snap = pipeline_snapshot.get_solid_def_snap('fancy_solid')
recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap(solid_def_snap)
snapshot.assert_match(serialize_pp(snap_from_config_type(recevied_config_type)))
_array_has_stable_hashes(
recevied_config_type, pipeline_snapshot.config_schema_snapshot.all_config_snaps_by_key
)
def test_multi_type_config_array_map(snapshot):
@solid(config_schema=Array(Map(str, int)))
def fancy_solid(_):
pass
@pipeline
def noop_pipeline():
fancy_solid()
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline)
solid_def_snap = pipeline_snapshot.get_node_def_snap("fancy_solid")
recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap(solid_def_snap)
snapshot.assert_match(serialize_pp(snap_from_config_type(recevied_config_type)))
_array_has_stable_hashes(
recevied_config_type, pipeline_snapshot.config_schema_snapshot.all_config_snaps_by_key
)
def config_type(cls):
return {
"max_concurrent_runs":
Field(
config=IntSource,
is_required=False,
description=
"The maximum number of runs that are allowed to be in progress at once. "
"Defaults to 10. Set to -1 to disable the limit. Set to 0 to stop any runs from launching. "
"Any other negative values are disallowed.",
),
"tag_concurrency_limits":
Field(
config=Noneable(
Array(
Shape({
"key":
String,
"value":
Field(
ScalarUnion(
scalar_type=String,
non_scalar_schema=Shape(
{"applyLimitPerUniqueValue": Bool}),
),
is_required=False,
),
"limit":
Field(int),
}))),
is_required=False,
description=
"A set of limits that are applied to runs with particular tags. "
"If a value is set, the limit is applied to only that key-value pair. "
"If no value is set, the limit is applied across all values of that key. "
"If the value is set to a dict with `applyLimitPerUniqueValue: true`, the limit "
"will apply to the number of unique values for that key.",
),
"dequeue_interval_seconds":
Field(
config=IntSource,
is_required=False,
description=
"The interval in seconds at which the Dagster Daemon "
"should periodically check the run queue for new runs to launch.",
),
}
def get_outputs_field(solid, handle):
check.inst_param(solid, "solid", Solid)
check.inst_param(handle, "handle", SolidHandle)
solid_def = solid.definition
if not solid_def.has_configurable_outputs:
return None
output_dict_fields = {}
for name, out in solid_def.output_dict.items():
if out.dagster_type.materializer:
output_dict_fields[name] = Field(
out.dagster_type.materializer.schema_type, is_required=False)
output_entry_dict = Shape(output_dict_fields)
return Field(Array(output_entry_dict), is_required=False)
def config_type(cls):
return {
"max_concurrent_runs":
Field(config=int, is_required=False),
"tag_concurrency_limits":
Field(
config=Noneable(
Array(
Shape({
"key": String,
"value": Field(String, is_required=False),
"limit": Field(int),
}))),
is_required=False,
),
"dequeue_interval_seconds":
Field(config=int, is_required=False),
}
def get_outputs_field(solid, handle):
check.inst_param(solid, 'solid', Solid)
check.inst_param(handle, 'handle', SolidHandle)
solid_def = solid.definition
if not solid_def.has_configurable_outputs:
return None
output_dict_fields = {}
for name, out in solid_def.output_dict.items():
if out.runtime_type.output_materialization_config:
output_dict_fields[name] = Field(
out.runtime_type.output_materialization_config.schema_type,
is_optional=True)
output_entry_dict = Shape(output_dict_fields)
return Field(Array(output_entry_dict), is_optional=True)
def test_deserialize_solid_def_snaps_multi_type_config(snapshot):
@solid(config=Field(
Permissive({
'foo':
Field(Array(float)),
'bar':
Selector({
'baz': Field(Noneable(int)),
'qux': {
'quux':
Field(str),
'corge':
Field(
Enum(
'RGB',
[
EnumValue('red'),
EnumValue('green'),
EnumValue('blue')
],
)),
},
}),
})))
def fancy_solid(_):
pass
@pipeline
def noop_pipeline():
fancy_solid()
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline)
solid_def_snap = pipeline_snapshot.get_solid_def_snap('fancy_solid')
recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap(
solid_def_snap)
snapshot.assert_match(
serialize_pp(snap_from_config_type(recevied_config_type)))
_map_has_stable_hashes(
recevied_config_type,
pipeline_snapshot.config_schema_snapshot.all_config_snaps_by_key)
def test_deserialize_solid_def_snaps_multi_type_config(snapshot):
@solid(config_schema=Field(
Permissive({
"foo":
Field(Array(float)),
"bar":
Selector({
"baz": Field(Noneable(int)),
"qux": {
"quux":
Field(str),
"corge":
Field(
Enum(
"RGB",
[
EnumValue("red"),
EnumValue("green"),
EnumValue("blue")
],
)),
},
}),
})))
def fancy_solid(_):
pass
@pipeline
def noop_pipeline():
fancy_solid()
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline)
solid_def_snap = pipeline_snapshot.get_solid_def_snap("fancy_solid")
recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap(
solid_def_snap)
snapshot.assert_match(
serialize_pp(snap_from_config_type(recevied_config_type)))
_map_has_stable_hashes(
recevied_config_type,
pipeline_snapshot.config_schema_snapshot.all_config_snaps_by_key)
def __init__(self, inner_dagster_type):
self._inner_dagster_type = check.inst_param(
inner_dagster_type, "inner_dagster_type", DagsterType
)
check.param_invariant(inner_dagster_type.loader, "inner_dagster_type")
self._schema_type = Array(inner_dagster_type.loader.schema_type)
def config_type(cls):
return Shape({"bad_run_ids": Field(Array(str), is_required=False)})
def schema_type(self):
return Array(ConfigAnyInstance)
def schema_type(self):
return Array(
self._item_dagster_type.input_hydration_config.schema_type)
def schema_type(self):
return Array(
self._item_runtime_type.input_hydration_config.schema_type)
'''This function remaps a python type to a Dagster type, or passes it through if it cannot be
remapped.
'''
from dagster.core.types.dagster_type import resolve_dagster_type
check.param_invariant(is_supported_runtime_python_builtin(ttype), 'ttype')
return resolve_dagster_type(SUPPORTED_RUNTIME_BUILTINS[ttype])
SUPPORTED_CONFIG_BUILTINS = {
int: Int,
float: Float,
bool: Bool,
str: String,
list: Array(ConfigAny),
dict: Permissive(),
}
def is_supported_config_python_builtin(ttype):
return ttype in SUPPORTED_CONFIG_BUILTINS
def remap_python_builtin_for_config(ttype):
'''This function remaps a python type to a Dagster type, or passes it through if it cannot be
remapped.
'''
from dagster.config.field import resolve_to_config_type
check.param_invariant(is_supported_config_python_builtin(ttype), 'ttype')
def schema_type(self):
return Array(self._item_dagster_type.loader.schema_type)
