def define_isolid_field(solid, handle, dependency_structure, pipeline_name): check.inst_param(solid, 'solid', Solid) check.inst_param(handle, 'handle', SolidHandle) check.str_param(pipeline_name, 'pipeline_name') if isinstance(solid.definition, CompositeSolidDefinition): composite_def = solid.definition child_solids_config_field = Field( define_solid_dictionary_cls( '{pipeline_name}.CompositeSolidsDict.{solid_handle}'.format( pipeline_name=camelcase(pipeline_name), solid_handle=handle.camelcase()), composite_def.solids, composite_def.dependency_structure, pipeline_name, handle, )) composite_config_dict = { 'inputs': get_inputs_field(solid, handle, dependency_structure, pipeline_name), 'outputs': get_outputs_field(solid, handle, pipeline_name), } # Mask solid config for solids beneath this level if config mapping is provided if composite_def.has_config_mapping: composite_config_dict[ 'config'] = composite_def.config_mapping.config_field else: composite_config_dict['solids'] = child_solids_config_field return Field( SolidContainerConfigDict( '{name}CompositeSolidConfig'.format(name=str(handle)), remove_none_entries(composite_config_dict), handle=handle, child_solids_config_field=child_solids_config_field, )) elif isinstance(solid.definition, SolidDefinition): solid_config_type = define_solid_config_cls( '{pipeline_name}.SolidConfig.{solid_handle}'.format( pipeline_name=camelcase(pipeline_name), solid_handle=handle.camelcase()), solid.definition.config_field, inputs_field=get_inputs_field(solid, handle, dependency_structure, pipeline_name), outputs_field=get_outputs_field(solid, handle, pipeline_name), ) return Field(solid_config_type) else: check.invariant('Unexpected ISolidDefinition type {type}'.format( type=type(solid.definition)))
def iterate_node_def_config_types(node_def: NodeDefinition) -> Iterator[ConfigType]: if isinstance(node_def, SolidDefinition): if node_def.has_config_field: yield from iterate_config_types(node_def.get_config_field().config_type) elif isinstance(node_def, GraphDefinition): for solid in node_def.solids: yield from iterate_node_def_config_types(solid.definition) else: check.invariant("Unexpected NodeDefinition type {type}".format(type=type(node_def)))
def iterate_node_def_config_types(node_def): check.inst_param(node_def, "node_def", NodeDefinition) if isinstance(node_def, SolidDefinition): if node_def.has_config_field: yield from iterate_config_types(node_def.config_field.config_type) elif isinstance(node_def, GraphDefinition): for solid in node_def.solids: yield from iterate_node_def_config_types(solid.definition) else: check.invariant("Unexpected NodeDefinition type {type}".format(type=type(node_def)))
def iterate_solid_def_types(solid_def): if isinstance(solid_def, SolidDefinition): if solid_def.config_field: for runtime_type in iterate_config_types(solid_def.config_field.config_type): yield runtime_type elif isinstance(solid_def, CompositeSolidDefinition): for solid in solid_def.solids: for def_type in iterate_solid_def_types(solid.definition): yield def_type else: check.invariant('Unexpected ISolidDefinition type {type}'.format(type=type(solid_def)))
def iterate_solid_def_config_types(solid_def): if isinstance(solid_def, SolidDefinition): if solid_def.config_schema: for config_type in iterate_config_types( solid_def.config_schema.config_type): yield config_type elif isinstance(solid_def, CompositeSolidDefinition): for solid in solid_def.solids: for config_type in iterate_solid_def_config_types( solid.definition): yield config_type else: check.invariant("Unexpected ISolidDefinition type {type}".format( type=type(solid_def)))
def remap_to_dagster_optional_type(type_annotation): check.invariant( is_python_optional_typehint(type_annotation), 'type must pass is_python_optional_typehint check', ) return WrappingNullableType(type_annotation.__args__[0])
def remap_to_dagster_list_type(type_annotation): check.invariant( is_python_list_typehint(type_annotation), 'type must pass is_python_list_typehint check' ) return WrappingListType(type_annotation.__args__[0])