def get_outputs_field( solid: Node, resource_defs: Dict[str, ResourceDefinition], ) -> Optional[Field]: # if any outputs have configurable output managers, use those for the schema and ignore all type # materializers output_manager_fields = {} for name, output_def in solid.definition.output_dict.items(): output_manager_output_field = get_output_manager_output_field( solid, output_def, resource_defs) if output_manager_output_field: output_manager_fields[name] = output_manager_output_field if output_manager_fields: return Field(Shape(output_manager_fields)) # otherwise, use any type materializers for the schema type_materializer_fields = {} for name, output_def in solid.definition.output_dict.items(): type_output_field = get_type_output_field(output_def) if type_output_field: type_materializer_fields[name] = type_output_field if type_materializer_fields: return Field(Array(Shape(type_materializer_fields)), is_required=False) return None
def get_outputs_field(solid, handle, resource_defs): check.inst_param(solid, "solid", Solid) check.inst_param(handle, "handle", SolidHandle) check.dict_param(resource_defs, "resource_defs", key_type=str, value_type=ResourceDefinition) # if any outputs have configurable output managers, use those for the schema and ignore all type # materializers output_manager_fields = {} for name, output_def in solid.definition.output_dict.items(): output_manager_output_field = get_output_manager_output_field( solid, output_def, resource_defs) if output_manager_output_field: output_manager_fields[name] = output_manager_output_field if output_manager_fields: return Field(Shape(output_manager_fields)) # otherwise, use any type materializers for the schema type_materializer_fields = {} for name, output_def in solid.definition.output_dict.items(): type_output_field = get_type_output_field(output_def) if type_output_field: type_materializer_fields[name] = type_output_field if type_materializer_fields: return Field(Array(Shape(type_materializer_fields)), is_required=False) return None
def config_type(cls): return { "max_concurrent_runs": Field(config=IntSource, is_required=False), "tag_concurrency_limits": Field( config=Noneable( Array( Shape({ "key": String, "value": Field( ScalarUnion( scalar_type=String, non_scalar_schema=Shape( {"applyLimitPerUniqueValue": Bool}), ), is_required=False, ), "limit": Field(int), }))), is_required=False, ), "dequeue_interval_seconds": Field(config=IntSource, is_required=False), }
def __init__(self, inner_dagster_type): self._inner_dagster_type = check.inst_param(inner_dagster_type, 'inner_dagster_type', DagsterType) check.param_invariant(inner_dagster_type.input_hydration_config, 'inner_dagster_type') self._schema_type = Array( inner_dagster_type.input_hydration_config.schema_type)
def _construct_array_from_snap(config_type_snap, config_snap_map): check.list_param(config_type_snap.type_param_keys, 'type_param_keys', str) check.invariant( len(config_type_snap.type_param_keys) == 1, 'Expect ARRAY to provide a single inner type. Snapshot provided: {}'. format(config_type_snap.type_param_keys), ) return Array(inner_type=construct_config_type_from_snap( config_snap_map[config_type_snap.type_param_keys[0]], config_snap_map))
def test_multi_type_config_array_dict_fields(dict_config_type, snapshot): @solid(config=Array(dict_config_type({'foo': Field(int), 'bar': Field(str)}))) def fancy_solid(_): pass @pipeline def noop_pipeline(): fancy_solid() pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline) solid_def_snap = pipeline_snapshot.get_solid_def_snap('fancy_solid') recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap(solid_def_snap) snapshot.assert_match(serialize_pp(snap_from_config_type(recevied_config_type))) _array_has_stable_hashes( recevied_config_type, pipeline_snapshot.config_schema_snapshot.all_config_snaps_by_key )
def test_multi_type_config_array_map(snapshot): @solid(config_schema=Array(Map(str, int))) def fancy_solid(_): pass @pipeline def noop_pipeline(): fancy_solid() pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline) solid_def_snap = pipeline_snapshot.get_node_def_snap("fancy_solid") recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap(solid_def_snap) snapshot.assert_match(serialize_pp(snap_from_config_type(recevied_config_type))) _array_has_stable_hashes( recevied_config_type, pipeline_snapshot.config_schema_snapshot.all_config_snaps_by_key )
def config_type(cls): return { "max_concurrent_runs": Field( config=IntSource, is_required=False, description= "The maximum number of runs that are allowed to be in progress at once. " "Defaults to 10. Set to -1 to disable the limit. Set to 0 to stop any runs from launching. " "Any other negative values are disallowed.", ), "tag_concurrency_limits": Field( config=Noneable( Array( Shape({ "key": String, "value": Field( ScalarUnion( scalar_type=String, non_scalar_schema=Shape( {"applyLimitPerUniqueValue": Bool}), ), is_required=False, ), "limit": Field(int), }))), is_required=False, description= "A set of limits that are applied to runs with particular tags. " "If a value is set, the limit is applied to only that key-value pair. " "If no value is set, the limit is applied across all values of that key. " "If the value is set to a dict with `applyLimitPerUniqueValue: true`, the limit " "will apply to the number of unique values for that key.", ), "dequeue_interval_seconds": Field( config=IntSource, is_required=False, description= "The interval in seconds at which the Dagster Daemon " "should periodically check the run queue for new runs to launch.", ), }
def get_outputs_field(solid, handle): check.inst_param(solid, "solid", Solid) check.inst_param(handle, "handle", SolidHandle) solid_def = solid.definition if not solid_def.has_configurable_outputs: return None output_dict_fields = {} for name, out in solid_def.output_dict.items(): if out.dagster_type.materializer: output_dict_fields[name] = Field( out.dagster_type.materializer.schema_type, is_required=False) output_entry_dict = Shape(output_dict_fields) return Field(Array(output_entry_dict), is_required=False)
def config_type(cls): return { "max_concurrent_runs": Field(config=int, is_required=False), "tag_concurrency_limits": Field( config=Noneable( Array( Shape({ "key": String, "value": Field(String, is_required=False), "limit": Field(int), }))), is_required=False, ), "dequeue_interval_seconds": Field(config=int, is_required=False), }
def get_outputs_field(solid, handle): check.inst_param(solid, 'solid', Solid) check.inst_param(handle, 'handle', SolidHandle) solid_def = solid.definition if not solid_def.has_configurable_outputs: return None output_dict_fields = {} for name, out in solid_def.output_dict.items(): if out.runtime_type.output_materialization_config: output_dict_fields[name] = Field( out.runtime_type.output_materialization_config.schema_type, is_optional=True) output_entry_dict = Shape(output_dict_fields) return Field(Array(output_entry_dict), is_optional=True)
def test_deserialize_solid_def_snaps_multi_type_config(snapshot): @solid(config=Field( Permissive({ 'foo': Field(Array(float)), 'bar': Selector({ 'baz': Field(Noneable(int)), 'qux': { 'quux': Field(str), 'corge': Field( Enum( 'RGB', [ EnumValue('red'), EnumValue('green'), EnumValue('blue') ], )), }, }), }))) def fancy_solid(_): pass @pipeline def noop_pipeline(): fancy_solid() pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline) solid_def_snap = pipeline_snapshot.get_solid_def_snap('fancy_solid') recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap( solid_def_snap) snapshot.assert_match( serialize_pp(snap_from_config_type(recevied_config_type))) _map_has_stable_hashes( recevied_config_type, pipeline_snapshot.config_schema_snapshot.all_config_snaps_by_key)
def test_deserialize_solid_def_snaps_multi_type_config(snapshot): @solid(config_schema=Field( Permissive({ "foo": Field(Array(float)), "bar": Selector({ "baz": Field(Noneable(int)), "qux": { "quux": Field(str), "corge": Field( Enum( "RGB", [ EnumValue("red"), EnumValue("green"), EnumValue("blue") ], )), }, }), }))) def fancy_solid(_): pass @pipeline def noop_pipeline(): fancy_solid() pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline) solid_def_snap = pipeline_snapshot.get_solid_def_snap("fancy_solid") recevied_config_type = pipeline_snapshot.get_config_type_from_solid_def_snap( solid_def_snap) snapshot.assert_match( serialize_pp(snap_from_config_type(recevied_config_type))) _map_has_stable_hashes( recevied_config_type, pipeline_snapshot.config_schema_snapshot.all_config_snaps_by_key)
def __init__(self, inner_dagster_type): self._inner_dagster_type = check.inst_param( inner_dagster_type, "inner_dagster_type", DagsterType ) check.param_invariant(inner_dagster_type.loader, "inner_dagster_type") self._schema_type = Array(inner_dagster_type.loader.schema_type)
def config_type(cls): return Shape({"bad_run_ids": Field(Array(str), is_required=False)})
def schema_type(self): return Array(ConfigAnyInstance)
def schema_type(self): return Array( self._item_dagster_type.input_hydration_config.schema_type)
def schema_type(self): return Array( self._item_runtime_type.input_hydration_config.schema_type)
'''This function remaps a python type to a Dagster type, or passes it through if it cannot be remapped. ''' from dagster.core.types.dagster_type import resolve_dagster_type check.param_invariant(is_supported_runtime_python_builtin(ttype), 'ttype') return resolve_dagster_type(SUPPORTED_RUNTIME_BUILTINS[ttype]) SUPPORTED_CONFIG_BUILTINS = { int: Int, float: Float, bool: Bool, str: String, list: Array(ConfigAny), dict: Permissive(), } def is_supported_config_python_builtin(ttype): return ttype in SUPPORTED_CONFIG_BUILTINS def remap_python_builtin_for_config(ttype): '''This function remaps a python type to a Dagster type, or passes it through if it cannot be remapped. ''' from dagster.config.field import resolve_to_config_type check.param_invariant(is_supported_config_python_builtin(ttype), 'ttype')
def schema_type(self): return Array(self._item_dagster_type.loader.schema_type)