def define_isolid_field(solid, handle, dependency_structure, pipeline_name):
check.inst_param(solid, 'solid', Solid)
check.inst_param(handle, 'handle', SolidHandle)
check.str_param(pipeline_name, 'pipeline_name')
if isinstance(solid.definition, CompositeSolidDefinition):
composite_def = solid.definition
child_solids_config_field = Field(
define_solid_dictionary_cls(
'{pipeline_name}.CompositeSolidsDict.{solid_handle}'.format(
pipeline_name=camelcase(pipeline_name),
solid_handle=handle.camelcase()),
composite_def.solids,
composite_def.dependency_structure,
pipeline_name,
handle,
))
composite_config_dict = {
'inputs':
get_inputs_field(solid, handle, dependency_structure,
pipeline_name),
'outputs':
get_outputs_field(solid, handle, pipeline_name),
}
# Mask solid config for solids beneath this level if config mapping is provided
if composite_def.has_config_mapping:
composite_config_dict[
'config'] = composite_def.config_mapping.config_field
else:
composite_config_dict['solids'] = child_solids_config_field
return Field(
SolidContainerConfigDict(
'{name}CompositeSolidConfig'.format(name=str(handle)),
remove_none_entries(composite_config_dict),
handle=handle,
child_solids_config_field=child_solids_config_field,
))
elif isinstance(solid.definition, SolidDefinition):
solid_config_type = define_solid_config_cls(
'{pipeline_name}.SolidConfig.{solid_handle}'.format(
pipeline_name=camelcase(pipeline_name),
solid_handle=handle.camelcase()),
solid.definition.config_field,
inputs_field=get_inputs_field(solid, handle, dependency_structure,
pipeline_name),
outputs_field=get_outputs_field(solid, handle, pipeline_name),
)
return Field(solid_config_type)
else:
check.invariant('Unexpected ISolidDefinition type {type}'.format(
type=type(solid.definition)))
def iterate_node_def_config_types(node_def: NodeDefinition) -> Iterator[ConfigType]:
if isinstance(node_def, SolidDefinition):
if node_def.has_config_field:
yield from iterate_config_types(node_def.get_config_field().config_type)
elif isinstance(node_def, GraphDefinition):
for solid in node_def.solids:
yield from iterate_node_def_config_types(solid.definition)
else:
check.invariant("Unexpected NodeDefinition type {type}".format(type=type(node_def)))
def iterate_node_def_config_types(node_def):
check.inst_param(node_def, "node_def", NodeDefinition)
if isinstance(node_def, SolidDefinition):
if node_def.has_config_field:
yield from iterate_config_types(node_def.config_field.config_type)
elif isinstance(node_def, GraphDefinition):
for solid in node_def.solids:
yield from iterate_node_def_config_types(solid.definition)
else:
check.invariant("Unexpected NodeDefinition type {type}".format(type=type(node_def)))
def iterate_solid_def_types(solid_def):
if isinstance(solid_def, SolidDefinition):
if solid_def.config_field:
for runtime_type in iterate_config_types(solid_def.config_field.config_type):
yield runtime_type
elif isinstance(solid_def, CompositeSolidDefinition):
for solid in solid_def.solids:
for def_type in iterate_solid_def_types(solid.definition):
yield def_type
else:
check.invariant('Unexpected ISolidDefinition type {type}'.format(type=type(solid_def)))
def iterate_solid_def_config_types(solid_def):
if isinstance(solid_def, SolidDefinition):
if solid_def.config_schema:
for config_type in iterate_config_types(
solid_def.config_schema.config_type):
yield config_type
elif isinstance(solid_def, CompositeSolidDefinition):
for solid in solid_def.solids:
for config_type in iterate_solid_def_config_types(
solid.definition):
yield config_type
else:
check.invariant("Unexpected ISolidDefinition type {type}".format(
type=type(solid_def)))
def remap_to_dagster_optional_type(type_annotation):
check.invariant(
is_python_optional_typehint(type_annotation),
'type must pass is_python_optional_typehint check',
)
return WrappingNullableType(type_annotation.__args__[0])
def remap_to_dagster_list_type(type_annotation):
check.invariant(
is_python_list_typehint(type_annotation), 'type must pass is_python_list_typehint check'
)
return WrappingListType(type_annotation.__args__[0])
