def _gather_all_schemas(node_defs: List[NodeDefinition]) -> Iterator[ConfigType]: dagster_types = construct_dagster_type_dictionary(node_defs) for dagster_type in list(dagster_types.values()) + list(ALL_RUNTIME_BUILTINS): if dagster_type.loader: yield from iterate_config_types(dagster_type.loader.schema_type) if dagster_type.materializer: yield from iterate_config_types(dagster_type.materializer.schema_type)
def _gather_all_schemas(solid_defs): dagster_types = construct_dagster_type_dictionary(solid_defs) for dagster_type in list(dagster_types.values()) + list(ALL_RUNTIME_BUILTINS): if dagster_type.loader: for ct in iterate_config_types(dagster_type.loader.schema_type): yield ct if dagster_type.materializer: for ct in iterate_config_types(dagster_type.materializer.schema_type): yield ct
def _gather_all_schemas(solid_defs): runtime_types = construct_dagster_type_dictionary(solid_defs) for rtt in runtime_types.values(): if rtt.input_hydration_config: for ct in iterate_config_types(rtt.input_hydration_config.schema_type): yield ct if rtt.output_materialization_config: for ct in iterate_config_types(rtt.output_materialization_config.schema_type): yield ct
def _gather_all_schemas(solid_defs): dagster_types = construct_dagster_type_dictionary(solid_defs) for dagster_type in list(dagster_types.values()) + list(ALL_RUNTIME_BUILTINS): if dagster_type.input_hydration_config: for ct in iterate_config_types(dagster_type.input_hydration_config.schema_type): yield ct if dagster_type.output_materialization_config: for ct in iterate_config_types(dagster_type.output_materialization_config.schema_type): yield ct
def __init__( self, name: str, description: Optional[str], node_defs: List[NodeDefinition], dependencies: Optional[Dict[Union[str, SolidInvocation], Dict[str, IDependencyDefinition]]], input_mappings: Optional[List[InputMapping]], output_mappings: Optional[List[OutputMapping]], config_mapping: Optional[ConfigMapping], **kwargs, ): self._node_defs = _check_node_defs_arg(name, node_defs) # TODO: backcompat for now self._solid_defs = self._node_defs self._dagster_type_dict = construct_dagster_type_dictionary( self._node_defs) self._dependencies = validate_dependency_dict(dependencies) self._dependency_structure, self._solid_dict = create_execution_structure( node_defs, self._dependencies, graph_definition=self) # List[InputMapping] self._input_mappings, input_defs = _validate_in_mappings( check.opt_list_param(input_mappings, "input_mappings"), self._solid_dict, self._dependency_structure, name, class_name=type(self).__name__, ) # List[OutputMapping] self._output_mappings = _validate_out_mappings( check.opt_list_param(output_mappings, "output_mappings"), self._solid_dict, self._dependency_structure, name, class_name=type(self).__name__, ) self._config_mapping = check.opt_inst_param(config_mapping, "config_mapping", ConfigMapping) super(GraphDefinition, self).__init__( name=name, description=description, input_defs=input_defs, output_defs=[ output_mapping.definition for output_mapping in self._output_mappings ], **kwargs, ) # must happen after base class construction as properties are assumed to be there # eager computation to detect cycles self.solids_in_topological_order = self._solids_in_topological_order()
def __init__( self, solid_defs, name=None, description=None, dependencies=None, mode_defs=None, preset_defs=None, tags=None, _parent_pipeline_def=None, # https://github.com/dagster-io/dagster/issues/2115 ): self._name = check.opt_str_param(name, 'name', '<<unnamed>>') self._description = check.opt_str_param(description, 'description') mode_definitions = check.opt_list_param(mode_defs, 'mode_defs', of_type=ModeDefinition) if not mode_definitions: mode_definitions = [ModeDefinition()] self._mode_definitions = mode_definitions self._current_level_solid_defs = check.list_param( _check_solids_arg(self._name, solid_defs), 'solid_defs', of_type=ISolidDefinition) self._tags = validate_tags(tags) seen_modes = set() for mode_def in mode_definitions: if mode_def.name in seen_modes: raise DagsterInvalidDefinitionError(( 'Two modes seen with the name "{mode_name}" in "{pipeline_name}". ' 'Modes must have unique names.').format( mode_name=mode_def.name, pipeline_name=self._name)) seen_modes.add(mode_def.name) self._dependencies = validate_dependency_dict(dependencies) dependency_structure, solid_dict = create_execution_structure( self._current_level_solid_defs, self._dependencies, container_definition=None) self._solid_dict = solid_dict self._dependency_structure = dependency_structure # eager toposort solids to detect cycles self.solids_in_topological_order = self._solids_in_topological_order() self._dagster_type_dict = construct_dagster_type_dictionary( self._current_level_solid_defs) self._preset_defs = check.opt_list_param(preset_defs, 'preset_defs', PresetDefinition) self._preset_dict = {} for preset in self._preset_defs: if preset.name in self._preset_dict: raise DagsterInvalidDefinitionError(( 'Two PresetDefinitions seen with the name "{name}" in "{pipeline_name}". ' 'PresetDefinitions must have unique names.').format( name=preset.name, pipeline_name=self._name)) if preset.mode not in seen_modes: raise DagsterInvalidDefinitionError( ('PresetDefinition "{name}" in "{pipeline_name}" ' 'references mode "{mode}" which is not defined.').format( name=preset.name, pipeline_name=self._name, mode=preset.mode)) self._preset_dict[preset.name] = preset # Validate solid resource dependencies _validate_resource_dependencies(self._mode_definitions, self._current_level_solid_defs) # Validate unsatisfied inputs can be materialized from config _validate_inputs(self._dependency_structure, self._solid_dict) self._all_solid_defs = _build_all_solid_defs( self._current_level_solid_defs) self._parent_pipeline_def = check.opt_inst_param( _parent_pipeline_def, '_parent_pipeline_def', PipelineDefinition) self._cached_enviroment_schemas = {} self._cached_pipeline_index = None
def __init__( self, solid_defs, name=None, description=None, dependencies=None, mode_defs=None, preset_defs=None, tags=None, hook_defs=None, input_mappings=None, output_mappings=None, config_mapping=None, positional_inputs=None, _parent_pipeline_def=None, # https://github.com/dagster-io/dagster/issues/2115 _configured_config_mapping_fn=None, _configured_config_schema=None, ): if not name: warnings.warn( "Pipeline must have a name. Names will be required starting in 0.10.0 or later." ) name = _anonymous_pipeline_name() # For these warnings they check truthiness because they get changed to [] higher # in the stack for the decorator case if input_mappings: experimental_arg_warning("input_mappings", "PipelineDefinition") if output_mappings: experimental_arg_warning("output_mappings", "PipelineDefinition") if config_mapping is not None: experimental_arg_warning("config_mapping", "PipelineDefinition") if positional_inputs: experimental_arg_warning("positional_inputs", "PipelineDefinition") super(PipelineDefinition, self).__init__( name=name, description=description, dependencies=dependencies, node_defs=solid_defs, tags=check.opt_dict_param(tags, "tags", key_type=str), positional_inputs=positional_inputs, input_mappings=input_mappings, output_mappings=output_mappings, config_mapping=config_mapping, _configured_config_mapping_fn=_configured_config_mapping_fn, _configured_config_schema=_configured_config_schema, ) self._current_level_node_defs = solid_defs self._tags = validate_tags(tags) mode_definitions = check.opt_list_param(mode_defs, "mode_defs", of_type=ModeDefinition) if not mode_definitions: mode_definitions = [ModeDefinition()] self._mode_definitions = mode_definitions seen_modes = set() for mode_def in mode_definitions: if mode_def.name in seen_modes: raise DagsterInvalidDefinitionError(( 'Two modes seen with the name "{mode_name}" in "{pipeline_name}". ' "Modes must have unique names.").format( mode_name=mode_def.name, pipeline_name=self._name)) seen_modes.add(mode_def.name) self._dagster_type_dict = construct_dagster_type_dictionary( self._current_level_node_defs) self._hook_defs = check.opt_set_param(hook_defs, "hook_defs", of_type=HookDefinition) self._preset_defs = check.opt_list_param(preset_defs, "preset_defs", PresetDefinition) self._preset_dict = {} for preset in self._preset_defs: if preset.name in self._preset_dict: raise DagsterInvalidDefinitionError(( 'Two PresetDefinitions seen with the name "{name}" in "{pipeline_name}". ' "PresetDefinitions must have unique names.").format( name=preset.name, pipeline_name=self._name)) if preset.mode not in seen_modes: raise DagsterInvalidDefinitionError( ('PresetDefinition "{name}" in "{pipeline_name}" ' 'references mode "{mode}" which is not defined.').format( name=preset.name, pipeline_name=self._name, mode=preset.mode)) self._preset_dict[preset.name] = preset # Validate solid resource dependencies _validate_resource_dependencies( self._mode_definitions, self._current_level_node_defs, self._dagster_type_dict, self._solid_dict, self._hook_defs, ) # Validate unsatisfied inputs can be materialized from config _validate_inputs(self._dependency_structure, self._solid_dict) # Recursively explore all nodes in the this pipeline self._all_node_defs = _build_all_node_defs( self._current_level_node_defs) self._parent_pipeline_def = check.opt_inst_param( _parent_pipeline_def, "_parent_pipeline_def", PipelineDefinition) self._cached_run_config_schemas = {} self._cached_external_pipeline = None
def __init__( self, solid_defs, name=None, description=None, dependencies=None, mode_defs=None, preset_defs=None, tags=None, hook_defs=None, _parent_pipeline_def=None, # https://github.com/dagster-io/dagster/issues/2115 ): if not name: warnings.warn( "Pipeline must have a name. Names will be required starting in 0.10.0 or later." ) self._name = check_valid_name( name) if name else _anonymous_pipeline_name() self._description = check.opt_str_param(description, "description") mode_definitions = check.opt_list_param(mode_defs, "mode_defs", of_type=ModeDefinition) if not mode_definitions: mode_definitions = [ModeDefinition()] self._mode_definitions = mode_definitions self._current_level_solid_defs = check.list_param( _check_solids_arg(self._name, solid_defs), "solid_defs", of_type=ISolidDefinition) self._tags = validate_tags(tags) seen_modes = set() for mode_def in mode_definitions: if mode_def.name in seen_modes: raise DagsterInvalidDefinitionError(( 'Two modes seen with the name "{mode_name}" in "{pipeline_name}". ' "Modes must have unique names.").format( mode_name=mode_def.name, pipeline_name=self._name)) seen_modes.add(mode_def.name) self._dependencies = validate_dependency_dict(dependencies) dependency_structure, solid_dict = create_execution_structure( self._current_level_solid_defs, self._dependencies, container_definition=None) self._solid_dict = solid_dict self._dependency_structure = dependency_structure # eager toposort solids to detect cycles self.solids_in_topological_order = self._solids_in_topological_order() self._dagster_type_dict = construct_dagster_type_dictionary( self._current_level_solid_defs) self._hook_defs = check.opt_set_param(hook_defs, "hook_defs", of_type=HookDefinition) self._preset_defs = check.opt_list_param(preset_defs, "preset_defs", PresetDefinition) self._preset_dict = {} for preset in self._preset_defs: if preset.name in self._preset_dict: raise DagsterInvalidDefinitionError(( 'Two PresetDefinitions seen with the name "{name}" in "{pipeline_name}". ' "PresetDefinitions must have unique names.").format( name=preset.name, pipeline_name=self._name)) if preset.mode not in seen_modes: raise DagsterInvalidDefinitionError( ('PresetDefinition "{name}" in "{pipeline_name}" ' 'references mode "{mode}" which is not defined.').format( name=preset.name, pipeline_name=self._name, mode=preset.mode)) self._preset_dict[preset.name] = preset # Validate solid resource dependencies _validate_resource_dependencies( self._mode_definitions, self._current_level_solid_defs, self._solid_dict, self._hook_defs, ) # Validate unsatisfied inputs can be materialized from config _validate_inputs(self._dependency_structure, self._solid_dict) self._all_solid_defs = _build_all_solid_defs( self._current_level_solid_defs) self._parent_pipeline_def = check.opt_inst_param( _parent_pipeline_def, "_parent_pipeline_def", PipelineDefinition) self._cached_run_config_schemas = {} self._cached_external_pipeline = None
def __init__( self, solid_defs, name=None, description=None, dependencies=None, mode_defs=None, preset_defs=None, ): self._name = check.opt_str_param(name, 'name', '<<unnamed>>') self._description = check.opt_str_param(description, 'description') mode_definitions = check.opt_list_param(mode_defs, 'mode_defs', of_type=ModeDefinition) if not mode_definitions: mode_definitions = [ModeDefinition()] self._mode_definitions = mode_definitions self._current_level_solid_defs = check.list_param( _check_solids_arg(self._name, solid_defs), 'solid_defs', of_type=ISolidDefinition) seen_modes = set() for mode_def in mode_definitions: if mode_def.name in seen_modes: raise DagsterInvalidDefinitionError(( 'Two modes seen with the name "{mode_name}" in "{pipeline_name}". ' 'Modes must have unique names.').format( mode_name=mode_def.name, pipeline_name=self._name)) seen_modes.add(mode_def.name) self._dependencies = validate_dependency_dict(dependencies) dependency_structure, solid_dict = create_execution_structure( self._current_level_solid_defs, self._dependencies, container_definition=None) self._solid_dict = solid_dict self._dependency_structure = dependency_structure self._runtime_type_dict = construct_dagster_type_dictionary( self._current_level_solid_defs) self._preset_defs = check.opt_list_param(preset_defs, 'preset_defs', PresetDefinition) self._preset_dict = {} for preset in self._preset_defs: if preset.name in self._preset_dict: raise DagsterInvalidDefinitionError(( 'Two PresetDefinitions seen with the name "{name}" in "{pipeline_name}". ' 'PresetDefinitions must have unique names.').format( name=preset.name, pipeline_name=self._name)) if preset.mode not in seen_modes: raise DagsterInvalidDefinitionError( ('PresetDefinition "{name}" in "{pipeline_name}" ' 'references mode "{mode}" which is not defined.').format( name=preset.name, pipeline_name=self._name, mode=preset.mode)) self._preset_dict[preset.name] = preset # Validate solid resource dependencies _validate_resource_dependencies(self._mode_definitions, self._current_level_solid_defs) # Validate unsatisfied inputs can be materialized from config _validate_inputs(self._dependency_structure, self._solid_dict) self._all_solid_defs = _build_all_solid_defs( self._current_level_solid_defs) self._selector = ExecutionSelector(self.name, list(solid_dict.keys())) self._cached_enviroment_schemas = {}