def _gather_all_schemas(node_defs: List[NodeDefinition]) -> Iterator[ConfigType]:
dagster_types = construct_dagster_type_dictionary(node_defs)
for dagster_type in list(dagster_types.values()) + list(ALL_RUNTIME_BUILTINS):
if dagster_type.loader:
yield from iterate_config_types(dagster_type.loader.schema_type)
if dagster_type.materializer:
yield from iterate_config_types(dagster_type.materializer.schema_type)
def _gather_all_schemas(solid_defs):
dagster_types = construct_dagster_type_dictionary(solid_defs)
for dagster_type in list(dagster_types.values()) + list(ALL_RUNTIME_BUILTINS):
if dagster_type.loader:
for ct in iterate_config_types(dagster_type.loader.schema_type):
yield ct
if dagster_type.materializer:
for ct in iterate_config_types(dagster_type.materializer.schema_type):
yield ct
def _gather_all_schemas(solid_defs):
runtime_types = construct_dagster_type_dictionary(solid_defs)
for rtt in runtime_types.values():
if rtt.input_hydration_config:
for ct in iterate_config_types(rtt.input_hydration_config.schema_type):
yield ct
if rtt.output_materialization_config:
for ct in iterate_config_types(rtt.output_materialization_config.schema_type):
yield ct
def _gather_all_schemas(solid_defs):
dagster_types = construct_dagster_type_dictionary(solid_defs)
for dagster_type in list(dagster_types.values()) + list(ALL_RUNTIME_BUILTINS):
if dagster_type.input_hydration_config:
for ct in iterate_config_types(dagster_type.input_hydration_config.schema_type):
yield ct
if dagster_type.output_materialization_config:
for ct in iterate_config_types(dagster_type.output_materialization_config.schema_type):
yield ct
def __init__(
self,
name: str,
description: Optional[str],
node_defs: List[NodeDefinition],
dependencies: Optional[Dict[Union[str, SolidInvocation],
Dict[str, IDependencyDefinition]]],
input_mappings: Optional[List[InputMapping]],
output_mappings: Optional[List[OutputMapping]],
config_mapping: Optional[ConfigMapping],
**kwargs,
):
self._node_defs = _check_node_defs_arg(name, node_defs)
# TODO: backcompat for now
self._solid_defs = self._node_defs
self._dagster_type_dict = construct_dagster_type_dictionary(
self._node_defs)
self._dependencies = validate_dependency_dict(dependencies)
self._dependency_structure, self._solid_dict = create_execution_structure(
node_defs, self._dependencies, graph_definition=self)
# List[InputMapping]
self._input_mappings, input_defs = _validate_in_mappings(
check.opt_list_param(input_mappings, "input_mappings"),
self._solid_dict,
self._dependency_structure,
name,
class_name=type(self).__name__,
)
# List[OutputMapping]
self._output_mappings = _validate_out_mappings(
check.opt_list_param(output_mappings, "output_mappings"),
self._solid_dict,
self._dependency_structure,
name,
class_name=type(self).__name__,
)
self._config_mapping = check.opt_inst_param(config_mapping,
"config_mapping",
ConfigMapping)
super(GraphDefinition, self).__init__(
name=name,
description=description,
input_defs=input_defs,
output_defs=[
output_mapping.definition
for output_mapping in self._output_mappings
],
**kwargs,
)
# must happen after base class construction as properties are assumed to be there
# eager computation to detect cycles
self.solids_in_topological_order = self._solids_in_topological_order()
def __init__(
self,
solid_defs,
name=None,
description=None,
dependencies=None,
mode_defs=None,
preset_defs=None,
tags=None,
_parent_pipeline_def=None, # https://github.com/dagster-io/dagster/issues/2115
):
self._name = check.opt_str_param(name, 'name', '<<unnamed>>')
self._description = check.opt_str_param(description, 'description')
mode_definitions = check.opt_list_param(mode_defs,
'mode_defs',
of_type=ModeDefinition)
if not mode_definitions:
mode_definitions = [ModeDefinition()]
self._mode_definitions = mode_definitions
self._current_level_solid_defs = check.list_param(
_check_solids_arg(self._name, solid_defs),
'solid_defs',
of_type=ISolidDefinition)
self._tags = validate_tags(tags)
seen_modes = set()
for mode_def in mode_definitions:
if mode_def.name in seen_modes:
raise DagsterInvalidDefinitionError((
'Two modes seen with the name "{mode_name}" in "{pipeline_name}". '
'Modes must have unique names.').format(
mode_name=mode_def.name, pipeline_name=self._name))
seen_modes.add(mode_def.name)
self._dependencies = validate_dependency_dict(dependencies)
dependency_structure, solid_dict = create_execution_structure(
self._current_level_solid_defs,
self._dependencies,
container_definition=None)
self._solid_dict = solid_dict
self._dependency_structure = dependency_structure
# eager toposort solids to detect cycles
self.solids_in_topological_order = self._solids_in_topological_order()
self._dagster_type_dict = construct_dagster_type_dictionary(
self._current_level_solid_defs)
self._preset_defs = check.opt_list_param(preset_defs, 'preset_defs',
PresetDefinition)
self._preset_dict = {}
for preset in self._preset_defs:
if preset.name in self._preset_dict:
raise DagsterInvalidDefinitionError((
'Two PresetDefinitions seen with the name "{name}" in "{pipeline_name}". '
'PresetDefinitions must have unique names.').format(
name=preset.name, pipeline_name=self._name))
if preset.mode not in seen_modes:
raise DagsterInvalidDefinitionError(
('PresetDefinition "{name}" in "{pipeline_name}" '
'references mode "{mode}" which is not defined.').format(
name=preset.name,
pipeline_name=self._name,
mode=preset.mode))
self._preset_dict[preset.name] = preset
# Validate solid resource dependencies
_validate_resource_dependencies(self._mode_definitions,
self._current_level_solid_defs)
# Validate unsatisfied inputs can be materialized from config
_validate_inputs(self._dependency_structure, self._solid_dict)
self._all_solid_defs = _build_all_solid_defs(
self._current_level_solid_defs)
self._parent_pipeline_def = check.opt_inst_param(
_parent_pipeline_def, '_parent_pipeline_def', PipelineDefinition)
self._cached_enviroment_schemas = {}
self._cached_pipeline_index = None
def __init__(
self,
solid_defs,
name=None,
description=None,
dependencies=None,
mode_defs=None,
preset_defs=None,
tags=None,
hook_defs=None,
input_mappings=None,
output_mappings=None,
config_mapping=None,
positional_inputs=None,
_parent_pipeline_def=None, # https://github.com/dagster-io/dagster/issues/2115
_configured_config_mapping_fn=None,
_configured_config_schema=None,
):
if not name:
warnings.warn(
"Pipeline must have a name. Names will be required starting in 0.10.0 or later."
)
name = _anonymous_pipeline_name()
# For these warnings they check truthiness because they get changed to [] higher
# in the stack for the decorator case
if input_mappings:
experimental_arg_warning("input_mappings", "PipelineDefinition")
if output_mappings:
experimental_arg_warning("output_mappings", "PipelineDefinition")
if config_mapping is not None:
experimental_arg_warning("config_mapping", "PipelineDefinition")
if positional_inputs:
experimental_arg_warning("positional_inputs", "PipelineDefinition")
super(PipelineDefinition, self).__init__(
name=name,
description=description,
dependencies=dependencies,
node_defs=solid_defs,
tags=check.opt_dict_param(tags, "tags", key_type=str),
positional_inputs=positional_inputs,
input_mappings=input_mappings,
output_mappings=output_mappings,
config_mapping=config_mapping,
_configured_config_mapping_fn=_configured_config_mapping_fn,
_configured_config_schema=_configured_config_schema,
)
self._current_level_node_defs = solid_defs
self._tags = validate_tags(tags)
mode_definitions = check.opt_list_param(mode_defs,
"mode_defs",
of_type=ModeDefinition)
if not mode_definitions:
mode_definitions = [ModeDefinition()]
self._mode_definitions = mode_definitions
seen_modes = set()
for mode_def in mode_definitions:
if mode_def.name in seen_modes:
raise DagsterInvalidDefinitionError((
'Two modes seen with the name "{mode_name}" in "{pipeline_name}". '
"Modes must have unique names.").format(
mode_name=mode_def.name, pipeline_name=self._name))
seen_modes.add(mode_def.name)
self._dagster_type_dict = construct_dagster_type_dictionary(
self._current_level_node_defs)
self._hook_defs = check.opt_set_param(hook_defs,
"hook_defs",
of_type=HookDefinition)
self._preset_defs = check.opt_list_param(preset_defs, "preset_defs",
PresetDefinition)
self._preset_dict = {}
for preset in self._preset_defs:
if preset.name in self._preset_dict:
raise DagsterInvalidDefinitionError((
'Two PresetDefinitions seen with the name "{name}" in "{pipeline_name}". '
"PresetDefinitions must have unique names.").format(
name=preset.name, pipeline_name=self._name))
if preset.mode not in seen_modes:
raise DagsterInvalidDefinitionError(
('PresetDefinition "{name}" in "{pipeline_name}" '
'references mode "{mode}" which is not defined.').format(
name=preset.name,
pipeline_name=self._name,
mode=preset.mode))
self._preset_dict[preset.name] = preset
# Validate solid resource dependencies
_validate_resource_dependencies(
self._mode_definitions,
self._current_level_node_defs,
self._dagster_type_dict,
self._solid_dict,
self._hook_defs,
)
# Validate unsatisfied inputs can be materialized from config
_validate_inputs(self._dependency_structure, self._solid_dict)
# Recursively explore all nodes in the this pipeline
self._all_node_defs = _build_all_node_defs(
self._current_level_node_defs)
self._parent_pipeline_def = check.opt_inst_param(
_parent_pipeline_def, "_parent_pipeline_def", PipelineDefinition)
self._cached_run_config_schemas = {}
self._cached_external_pipeline = None
def __init__(
self,
solid_defs,
name=None,
description=None,
dependencies=None,
mode_defs=None,
preset_defs=None,
tags=None,
hook_defs=None,
_parent_pipeline_def=None, # https://github.com/dagster-io/dagster/issues/2115
):
if not name:
warnings.warn(
"Pipeline must have a name. Names will be required starting in 0.10.0 or later."
)
self._name = check_valid_name(
name) if name else _anonymous_pipeline_name()
self._description = check.opt_str_param(description, "description")
mode_definitions = check.opt_list_param(mode_defs,
"mode_defs",
of_type=ModeDefinition)
if not mode_definitions:
mode_definitions = [ModeDefinition()]
self._mode_definitions = mode_definitions
self._current_level_solid_defs = check.list_param(
_check_solids_arg(self._name, solid_defs),
"solid_defs",
of_type=ISolidDefinition)
self._tags = validate_tags(tags)
seen_modes = set()
for mode_def in mode_definitions:
if mode_def.name in seen_modes:
raise DagsterInvalidDefinitionError((
'Two modes seen with the name "{mode_name}" in "{pipeline_name}". '
"Modes must have unique names.").format(
mode_name=mode_def.name, pipeline_name=self._name))
seen_modes.add(mode_def.name)
self._dependencies = validate_dependency_dict(dependencies)
dependency_structure, solid_dict = create_execution_structure(
self._current_level_solid_defs,
self._dependencies,
container_definition=None)
self._solid_dict = solid_dict
self._dependency_structure = dependency_structure
# eager toposort solids to detect cycles
self.solids_in_topological_order = self._solids_in_topological_order()
self._dagster_type_dict = construct_dagster_type_dictionary(
self._current_level_solid_defs)
self._hook_defs = check.opt_set_param(hook_defs,
"hook_defs",
of_type=HookDefinition)
self._preset_defs = check.opt_list_param(preset_defs, "preset_defs",
PresetDefinition)
self._preset_dict = {}
for preset in self._preset_defs:
if preset.name in self._preset_dict:
raise DagsterInvalidDefinitionError((
'Two PresetDefinitions seen with the name "{name}" in "{pipeline_name}". '
"PresetDefinitions must have unique names.").format(
name=preset.name, pipeline_name=self._name))
if preset.mode not in seen_modes:
raise DagsterInvalidDefinitionError(
('PresetDefinition "{name}" in "{pipeline_name}" '
'references mode "{mode}" which is not defined.').format(
name=preset.name,
pipeline_name=self._name,
mode=preset.mode))
self._preset_dict[preset.name] = preset
# Validate solid resource dependencies
_validate_resource_dependencies(
self._mode_definitions,
self._current_level_solid_defs,
self._solid_dict,
self._hook_defs,
)
# Validate unsatisfied inputs can be materialized from config
_validate_inputs(self._dependency_structure, self._solid_dict)
self._all_solid_defs = _build_all_solid_defs(
self._current_level_solid_defs)
self._parent_pipeline_def = check.opt_inst_param(
_parent_pipeline_def, "_parent_pipeline_def", PipelineDefinition)
self._cached_run_config_schemas = {}
self._cached_external_pipeline = None
def __init__(
self,
solid_defs,
name=None,
description=None,
dependencies=None,
mode_defs=None,
preset_defs=None,
):
self._name = check.opt_str_param(name, 'name', '<<unnamed>>')
self._description = check.opt_str_param(description, 'description')
mode_definitions = check.opt_list_param(mode_defs,
'mode_defs',
of_type=ModeDefinition)
if not mode_definitions:
mode_definitions = [ModeDefinition()]
self._mode_definitions = mode_definitions
self._current_level_solid_defs = check.list_param(
_check_solids_arg(self._name, solid_defs),
'solid_defs',
of_type=ISolidDefinition)
seen_modes = set()
for mode_def in mode_definitions:
if mode_def.name in seen_modes:
raise DagsterInvalidDefinitionError((
'Two modes seen with the name "{mode_name}" in "{pipeline_name}". '
'Modes must have unique names.').format(
mode_name=mode_def.name, pipeline_name=self._name))
seen_modes.add(mode_def.name)
self._dependencies = validate_dependency_dict(dependencies)
dependency_structure, solid_dict = create_execution_structure(
self._current_level_solid_defs,
self._dependencies,
container_definition=None)
self._solid_dict = solid_dict
self._dependency_structure = dependency_structure
self._runtime_type_dict = construct_dagster_type_dictionary(
self._current_level_solid_defs)
self._preset_defs = check.opt_list_param(preset_defs, 'preset_defs',
PresetDefinition)
self._preset_dict = {}
for preset in self._preset_defs:
if preset.name in self._preset_dict:
raise DagsterInvalidDefinitionError((
'Two PresetDefinitions seen with the name "{name}" in "{pipeline_name}". '
'PresetDefinitions must have unique names.').format(
name=preset.name, pipeline_name=self._name))
if preset.mode not in seen_modes:
raise DagsterInvalidDefinitionError(
('PresetDefinition "{name}" in "{pipeline_name}" '
'references mode "{mode}" which is not defined.').format(
name=preset.name,
pipeline_name=self._name,
mode=preset.mode))
self._preset_dict[preset.name] = preset
# Validate solid resource dependencies
_validate_resource_dependencies(self._mode_definitions,
self._current_level_solid_defs)
# Validate unsatisfied inputs can be materialized from config
_validate_inputs(self._dependency_structure, self._solid_dict)
self._all_solid_defs = _build_all_solid_defs(
self._current_level_solid_defs)
self._selector = ExecutionSelector(self.name, list(solid_dict.keys()))
self._cached_enviroment_schemas = {}
