def get_outputs_field(solid, handle, pipeline_name): check.inst_param(solid, 'solid', Solid) check.inst_param(handle, 'handle', SolidHandle) check.str_param(pipeline_name, 'pipeline_name') solid_def = solid.definition if not solid_def.has_configurable_outputs: return None output_dict_fields = {} for name, out in solid_def.output_dict.items(): if out.runtime_type.output_materialization_config: output_dict_fields[name] = Field( type(out.runtime_type.output_materialization_config.schema_type), is_optional=True ) output_entry_dict = SystemNamedDict( '{pipeline_name}.{solid_handle}.Outputs'.format( pipeline_name=camelcase(pipeline_name), solid_handle=handle.camelcase() ), output_dict_fields, ) return Field(List[output_entry_dict], is_optional=True)
def get_inputs_field(solid, handle, dependency_structure, pipeline_name): check.inst_param(solid, 'solid', Solid) check.inst_param(handle, 'handle', SolidHandle) check.inst_param(dependency_structure, 'dependency_structure', DependencyStructure) check.str_param(pipeline_name, 'pipeline_name') if not solid.definition.has_configurable_inputs: return None inputs_field_fields = {} for name, inp in solid.definition.input_dict.items(): if inp.runtime_type.input_hydration_config: inp_handle = SolidInputHandle(solid, inp) # If this input is not satisfied by a dependency you must # provide it via config if not dependency_structure.has_deps(inp_handle) and not solid.container_maps_input( name ): inputs_field_fields[name] = FieldImpl( inp.runtime_type.input_hydration_config.schema_type ) if not inputs_field_fields: return None return Field( SystemNamedDict( '{pipeline_name}.{solid_handle}.Inputs'.format( pipeline_name=camelcase(pipeline_name), solid_handle=handle.camelcase() ), inputs_field_fields, ) )
def define_mode_resources_dictionary_cls(pipeline_name, mode_definition): check.str_param(pipeline_name, 'pipeline_name') check.inst_param(mode_definition, 'mode_definition', ModeDefinition) return define_resource_dictionary_cls( '{pipeline_name}.Mode.{mode}.Resources'.format( pipeline_name=pipeline_name, mode=camelcase(mode_definition.name)), mode_definition.resource_defs, )
def define_isolid_field(solid, handle, dependency_structure, pipeline_name): check.inst_param(solid, 'solid', Solid) check.inst_param(handle, 'handle', SolidHandle) check.str_param(pipeline_name, 'pipeline_name') if isinstance(solid.definition, CompositeSolidDefinition): composite_def = solid.definition child_solids_config_field = Field( define_solid_dictionary_cls( '{pipeline_name}.CompositeSolidsDict.{solid_handle}'.format( pipeline_name=camelcase(pipeline_name), solid_handle=handle.camelcase()), composite_def.solids, composite_def.dependency_structure, pipeline_name, handle, )) composite_config_dict = { 'inputs': get_inputs_field(solid, handle, dependency_structure, pipeline_name), 'outputs': get_outputs_field(solid, handle, pipeline_name), } # Mask solid config for solids beneath this level if config mapping is provided if composite_def.has_config_mapping: composite_config_dict[ 'config'] = composite_def.config_mapping.config_field else: composite_config_dict['solids'] = child_solids_config_field return Field( SolidContainerConfigDict( '{name}CompositeSolidConfig'.format(name=str(handle)), remove_none_entries(composite_config_dict), handle=handle, child_solids_config_field=child_solids_config_field, )) elif isinstance(solid.definition, SolidDefinition): solid_config_type = define_solid_config_cls( '{pipeline_name}.SolidConfig.{solid_handle}'.format( pipeline_name=camelcase(pipeline_name), solid_handle=handle.camelcase()), solid.definition.config_field, inputs_field=get_inputs_field(solid, handle, dependency_structure, pipeline_name), outputs_field=get_outputs_field(solid, handle, pipeline_name), ) return Field(solid_config_type) else: check.invariant('Unexpected ISolidDefinition type {type}'.format( type=type(solid.definition)))
def define_resource_dictionary_cls(name, resource_defs): check.str_param(name, 'name') check.dict_param(resource_defs, 'resource_defs', key_type=str, value_type=ResourceDefinition) fields = {} for resource_name, resource_def in resource_defs.items(): if resource_def.config_field: fields[resource_name] = Field(define_resource_cls(name, resource_name, resource_def)) return SystemNamedDict(name=name, fields=fields)
def define_solid_config_cls(name, config_field, inputs_field, outputs_field): check.str_param(name, 'name') check_opt_field_param(config_field, 'config_field') check_opt_field_param(inputs_field, 'inputs_field') check_opt_field_param(outputs_field, 'outputs_field') return SystemNamedDict( name, remove_none_entries( {'config': config_field, 'inputs': inputs_field, 'outputs': outputs_field} ), )
def __new__( cls, pipeline_name, solids, dependency_structure, mode_definition, logger_defs, ignored_solids, ): return super(EnvironmentClassCreationData, cls).__new__( cls, pipeline_name=check.str_param(pipeline_name, "pipeline_name"), solids=check.list_param(solids, "solids", of_type=Solid), dependency_structure=check.inst_param(dependency_structure, "dependency_structure", DependencyStructure), mode_definition=check.inst_param(mode_definition, "mode_definition", ModeDefinition), logger_defs=check.dict_param(logger_defs, "logger_defs", key_type=str, value_type=LoggerDefinition), ignored_solids=check.list_param(ignored_solids, "ignored_solids", of_type=Solid), )
def define_storage_config_cls(type_name, mode_definition): check.str_param(type_name, 'type_name') check.inst_param(mode_definition, 'mode_definition', ModeDefinition) fields = {} for storage_def in mode_definition.system_storage_defs: fields[storage_def.name] = Field( SystemNamedDict( name='{type_name}.{storage_name}'.format( type_name=type_name, storage_name=camelcase(storage_def.name) ), fields={'config': storage_def.config_field} if storage_def.config_field else {}, ) ) return SystemNamedSelector(type_name, fields)
def define_executor_config_cls(type_name, mode_definition): check.str_param(type_name, 'type_name') check.inst_param(mode_definition, 'mode_definition', ModeDefinition) fields = {} for executor_def in mode_definition.executor_defs: fields[executor_def.name] = Field( SystemNamedDict( name='{type_name}.{executor_name}'.format( type_name=type_name, executor_name=camelcase(executor_def.name) ), fields={'config': executor_def.config_field} if executor_def.config_field else {}, ) ) return SystemNamedSelector(type_name, fields)
def define_logger_dictionary_cls(name, creation_data): check.str_param(name, 'name') check.inst_param(creation_data, 'creation_data', EnvironmentClassCreationData) fields = {} for logger_name, logger_definition in creation_data.logger_defs.items(): fields[logger_name] = Field( SystemNamedDict( '{pipeline_name}.LoggerConfig.{logger_name}'.format( pipeline_name=camelcase(creation_data.pipeline_name), logger_name=camelcase(logger_name), ), remove_none_entries({'config': logger_definition.config_field}), ), is_optional=True, ) return SystemNamedDict(name, fields)
def define_solid_dictionary_cls( name, solids, dependency_structure, pipeline_name, parent_handle=None ): check.str_param(name, 'name') check.list_param(solids, 'solids', of_type=Solid) check.inst_param(dependency_structure, 'dependency_structure', DependencyStructure) check.str_param(pipeline_name, 'pipeline_name') check.opt_inst_param(parent_handle, 'parent_handle', SolidHandle) fields = {} for solid in solids: if solid.definition.has_config_entry: fields[solid.name] = define_isolid_field( solid, SolidHandle(solid.name, solid.definition.name, parent_handle), dependency_structure, pipeline_name, ) return SystemNamedDict(name, fields)
def __new__(cls, pipeline_name, solids, dependency_structure, mode_definition, logger_defs): return super(EnvironmentClassCreationData, cls).__new__( cls, pipeline_name=check.str_param(pipeline_name, 'pipeline_name'), solids=check.list_param(solids, 'solids', of_type=Solid), dependency_structure=check.inst_param( dependency_structure, 'dependency_structure', DependencyStructure ), mode_definition=check.inst_param(mode_definition, 'mode_definition', ModeDefinition), logger_defs=check.dict_param( logger_defs, 'logger_defs', key_type=str, value_type=LoggerDefinition ), )
def define_execution_config_cls(name): check.str_param(name, 'name') return SystemNamedDict(name, {})
def define_expectations_config_cls(name): check.str_param(name, 'name') return SystemNamedDict( name, fields={'evaluate': Field(Bool, is_optional=True, default_value=True)} )