def get_outputs_field(solid, handle, pipeline_name):
check.inst_param(solid, 'solid', Solid)
check.inst_param(handle, 'handle', SolidHandle)
check.str_param(pipeline_name, 'pipeline_name')
solid_def = solid.definition
if not solid_def.has_configurable_outputs:
return None
output_dict_fields = {}
for name, out in solid_def.output_dict.items():
if out.runtime_type.output_materialization_config:
output_dict_fields[name] = Field(
type(out.runtime_type.output_materialization_config.schema_type), is_optional=True
)
output_entry_dict = SystemNamedDict(
'{pipeline_name}.{solid_handle}.Outputs'.format(
pipeline_name=camelcase(pipeline_name), solid_handle=handle.camelcase()
),
output_dict_fields,
)
return Field(List[output_entry_dict], is_optional=True)
def get_inputs_field(solid, handle, dependency_structure, pipeline_name):
check.inst_param(solid, 'solid', Solid)
check.inst_param(handle, 'handle', SolidHandle)
check.inst_param(dependency_structure, 'dependency_structure', DependencyStructure)
check.str_param(pipeline_name, 'pipeline_name')
if not solid.definition.has_configurable_inputs:
return None
inputs_field_fields = {}
for name, inp in solid.definition.input_dict.items():
if inp.runtime_type.input_hydration_config:
inp_handle = SolidInputHandle(solid, inp)
# If this input is not satisfied by a dependency you must
# provide it via config
if not dependency_structure.has_deps(inp_handle) and not solid.container_maps_input(
name
):
inputs_field_fields[name] = FieldImpl(
inp.runtime_type.input_hydration_config.schema_type
)
if not inputs_field_fields:
return None
return Field(
SystemNamedDict(
'{pipeline_name}.{solid_handle}.Inputs'.format(
pipeline_name=camelcase(pipeline_name), solid_handle=handle.camelcase()
),
inputs_field_fields,
)
)
def define_mode_resources_dictionary_cls(pipeline_name, mode_definition):
check.str_param(pipeline_name, 'pipeline_name')
check.inst_param(mode_definition, 'mode_definition', ModeDefinition)
return define_resource_dictionary_cls(
'{pipeline_name}.Mode.{mode}.Resources'.format(
pipeline_name=pipeline_name, mode=camelcase(mode_definition.name)),
mode_definition.resource_defs,
)
def define_isolid_field(solid, handle, dependency_structure, pipeline_name):
check.inst_param(solid, 'solid', Solid)
check.inst_param(handle, 'handle', SolidHandle)
check.str_param(pipeline_name, 'pipeline_name')
if isinstance(solid.definition, CompositeSolidDefinition):
composite_def = solid.definition
child_solids_config_field = Field(
define_solid_dictionary_cls(
'{pipeline_name}.CompositeSolidsDict.{solid_handle}'.format(
pipeline_name=camelcase(pipeline_name),
solid_handle=handle.camelcase()),
composite_def.solids,
composite_def.dependency_structure,
pipeline_name,
handle,
))
composite_config_dict = {
'inputs':
get_inputs_field(solid, handle, dependency_structure,
pipeline_name),
'outputs':
get_outputs_field(solid, handle, pipeline_name),
}
# Mask solid config for solids beneath this level if config mapping is provided
if composite_def.has_config_mapping:
composite_config_dict[
'config'] = composite_def.config_mapping.config_field
else:
composite_config_dict['solids'] = child_solids_config_field
return Field(
SolidContainerConfigDict(
'{name}CompositeSolidConfig'.format(name=str(handle)),
remove_none_entries(composite_config_dict),
handle=handle,
child_solids_config_field=child_solids_config_field,
))
elif isinstance(solid.definition, SolidDefinition):
solid_config_type = define_solid_config_cls(
'{pipeline_name}.SolidConfig.{solid_handle}'.format(
pipeline_name=camelcase(pipeline_name),
solid_handle=handle.camelcase()),
solid.definition.config_field,
inputs_field=get_inputs_field(solid, handle, dependency_structure,
pipeline_name),
outputs_field=get_outputs_field(solid, handle, pipeline_name),
)
return Field(solid_config_type)
else:
check.invariant('Unexpected ISolidDefinition type {type}'.format(
type=type(solid.definition)))
def define_resource_dictionary_cls(name, resource_defs):
check.str_param(name, 'name')
check.dict_param(resource_defs, 'resource_defs', key_type=str, value_type=ResourceDefinition)
fields = {}
for resource_name, resource_def in resource_defs.items():
if resource_def.config_field:
fields[resource_name] = Field(define_resource_cls(name, resource_name, resource_def))
return SystemNamedDict(name=name, fields=fields)
def define_solid_config_cls(name, config_field, inputs_field, outputs_field):
check.str_param(name, 'name')
check_opt_field_param(config_field, 'config_field')
check_opt_field_param(inputs_field, 'inputs_field')
check_opt_field_param(outputs_field, 'outputs_field')
return SystemNamedDict(
name,
remove_none_entries(
{'config': config_field, 'inputs': inputs_field, 'outputs': outputs_field}
),
)
def __new__(
cls,
pipeline_name,
solids,
dependency_structure,
mode_definition,
logger_defs,
ignored_solids,
):
return super(EnvironmentClassCreationData, cls).__new__(
cls,
pipeline_name=check.str_param(pipeline_name, "pipeline_name"),
solids=check.list_param(solids, "solids", of_type=Solid),
dependency_structure=check.inst_param(dependency_structure,
"dependency_structure",
DependencyStructure),
mode_definition=check.inst_param(mode_definition,
"mode_definition",
ModeDefinition),
logger_defs=check.dict_param(logger_defs,
"logger_defs",
key_type=str,
value_type=LoggerDefinition),
ignored_solids=check.list_param(ignored_solids,
"ignored_solids",
of_type=Solid),
)
def define_storage_config_cls(type_name, mode_definition):
check.str_param(type_name, 'type_name')
check.inst_param(mode_definition, 'mode_definition', ModeDefinition)
fields = {}
for storage_def in mode_definition.system_storage_defs:
fields[storage_def.name] = Field(
SystemNamedDict(
name='{type_name}.{storage_name}'.format(
type_name=type_name, storage_name=camelcase(storage_def.name)
),
fields={'config': storage_def.config_field} if storage_def.config_field else {},
)
)
return SystemNamedSelector(type_name, fields)
def define_executor_config_cls(type_name, mode_definition):
check.str_param(type_name, 'type_name')
check.inst_param(mode_definition, 'mode_definition', ModeDefinition)
fields = {}
for executor_def in mode_definition.executor_defs:
fields[executor_def.name] = Field(
SystemNamedDict(
name='{type_name}.{executor_name}'.format(
type_name=type_name, executor_name=camelcase(executor_def.name)
),
fields={'config': executor_def.config_field} if executor_def.config_field else {},
)
)
return SystemNamedSelector(type_name, fields)
def define_logger_dictionary_cls(name, creation_data):
check.str_param(name, 'name')
check.inst_param(creation_data, 'creation_data', EnvironmentClassCreationData)
fields = {}
for logger_name, logger_definition in creation_data.logger_defs.items():
fields[logger_name] = Field(
SystemNamedDict(
'{pipeline_name}.LoggerConfig.{logger_name}'.format(
pipeline_name=camelcase(creation_data.pipeline_name),
logger_name=camelcase(logger_name),
),
remove_none_entries({'config': logger_definition.config_field}),
),
is_optional=True,
)
return SystemNamedDict(name, fields)
def define_solid_dictionary_cls(
name, solids, dependency_structure, pipeline_name, parent_handle=None
):
check.str_param(name, 'name')
check.list_param(solids, 'solids', of_type=Solid)
check.inst_param(dependency_structure, 'dependency_structure', DependencyStructure)
check.str_param(pipeline_name, 'pipeline_name')
check.opt_inst_param(parent_handle, 'parent_handle', SolidHandle)
fields = {}
for solid in solids:
if solid.definition.has_config_entry:
fields[solid.name] = define_isolid_field(
solid,
SolidHandle(solid.name, solid.definition.name, parent_handle),
dependency_structure,
pipeline_name,
)
return SystemNamedDict(name, fields)
def __new__(cls, pipeline_name, solids, dependency_structure, mode_definition, logger_defs):
return super(EnvironmentClassCreationData, cls).__new__(
cls,
pipeline_name=check.str_param(pipeline_name, 'pipeline_name'),
solids=check.list_param(solids, 'solids', of_type=Solid),
dependency_structure=check.inst_param(
dependency_structure, 'dependency_structure', DependencyStructure
),
mode_definition=check.inst_param(mode_definition, 'mode_definition', ModeDefinition),
logger_defs=check.dict_param(
logger_defs, 'logger_defs', key_type=str, value_type=LoggerDefinition
),
)
def define_execution_config_cls(name):
check.str_param(name, 'name')
return SystemNamedDict(name, {})
def define_expectations_config_cls(name):
check.str_param(name, 'name')
return SystemNamedDict(
name, fields={'evaluate': Field(Bool, is_optional=True, default_value=True)}
)
