def _gather_all_schemas(solid_defs):
runtime_types = construct_runtime_type_dictionary(solid_defs)
for rtt in runtime_types.values():
if rtt.input_schema:
for ct in iterate_config_types(rtt.input_schema.schema_type):
yield ct
if rtt.output_schema:
for ct in iterate_config_types(rtt.output_schema.schema_type):
yield ct
def __init__(
self,
solid_defs,
name=None,
description=None,
dependencies=None,
mode_defs=None,
preset_defs=None,
):
self._name = check.opt_str_param(name, 'name', '<<unnamed>>')
self._description = check.opt_str_param(description, 'description')
mode_definitions = check.opt_list_param(mode_defs,
'mode_defs',
of_type=ModeDefinition)
if not mode_definitions:
mode_definitions = [ModeDefinition()]
self._mode_definitions = mode_definitions
current_level_solid_defs = check.list_param(_check_solids_arg(
self._name, solid_defs),
'solid_defs',
of_type=ISolidDefinition)
seen_modes = set()
for mode_def in mode_definitions:
if mode_def.name in seen_modes:
raise DagsterInvalidDefinitionError((
'Two modes seen with the name "{mode_name}" in "{pipeline_name}". '
'Modes must have unique names.').format(
mode_name=mode_def.name, pipeline_name=self._name))
seen_modes.add(mode_def.name)
self._dependencies = validate_dependency_dict(dependencies)
dependency_structure, pipeline_solid_dict = create_execution_structure(
current_level_solid_defs,
self._dependencies,
container_definition=None)
self._solid_dict = pipeline_solid_dict
self._dependency_structure = dependency_structure
self._runtime_type_dict = construct_runtime_type_dictionary(
current_level_solid_defs)
self._preset_defs = check.opt_list_param(preset_defs, 'preset_defs',
PresetDefinition)
self._preset_dict = {}
for preset in self._preset_defs:
if preset.name in self._preset_dict:
raise DagsterInvalidDefinitionError((
'Two PresetDefinitions seen with the name "{name}" in "{pipeline_name}". '
'PresetDefinitions must have unique names.').format(
name=preset.name, pipeline_name=self._name))
if preset.mode not in seen_modes:
raise DagsterInvalidDefinitionError(
('PresetDefinition "{name}" in "{pipeline_name}" '
'references mode "{mode}" which is not defined.').format(
name=preset.name,
pipeline_name=self._name,
mode=preset.mode))
self._preset_dict[preset.name] = preset
# Validate solid resource dependencies
_validate_resource_dependencies(self._mode_definitions,
current_level_solid_defs)
self._all_solid_defs = {}
for current_level_solid_def in current_level_solid_defs:
for solid_def in current_level_solid_def.iterate_solid_defs():
self._all_solid_defs[solid_def.name] = solid_def
