def __init__( self, resource_fn=None, config_schema=None, description=None, _configured_config_mapping_fn=None, version=None, ): EXPECTED_POSITIONALS = ["*"] fn_positionals, _ = split_function_parameters(resource_fn, EXPECTED_POSITIONALS) missing_positional = validate_decorated_fn_positionals(fn_positionals, EXPECTED_POSITIONALS) if missing_positional: raise DagsterInvalidDefinitionError( "@resource '{resource_name}' decorated function does not have required " "positional parameter '{missing_param}'. Resource functions should only have keyword " "arguments that match input names and a first positional parameter.".format( resource_name=resource_fn.__name__, missing_param=missing_positional ) ) self._resource_fn = check.opt_callable_param(resource_fn, "resource_fn") self._config_schema = check_user_facing_opt_config_param(config_schema, "config_schema") self._description = check.opt_str_param(description, "description") self.__configured_config_mapping_fn = check.opt_callable_param( _configured_config_mapping_fn, "config_mapping_fn" ) self._version = check.opt_str_param(version, "version") if version: experimental_arg_warning("version", "ResourceDefinition.__init__")
def __new__( cls, dagster_type=NoValueSentinel, description=None, is_required=None, io_manager_key=None, metadata=None, asset_key=None, asset_partitions=None, asset_partitions_def=None, # make sure new parameters are updated in combine_with_inferred below ): if asset_partitions_def: experimental_arg_warning("assets_definition", "Out.__new__") return super(Out, cls).__new__( cls, dagster_type=dagster_type, description=description, is_required=is_required, io_manager_key=io_manager_key, metadata=metadata, asset_key=asset_key, asset_partitions=asset_partitions, asset_partitions_def=asset_partitions_def, )
def __init__( self, name, dagster_type=None, description=None, default_value=_NoValueSentinel, root_manager_key=None, metadata=None, ): "" self._name = check_valid_name(name) self._dagster_type = check.inst(resolve_dagster_type(dagster_type), DagsterType) self._description = check.opt_str_param(description, "description") self._default_value = _check_default_value(self._name, self._dagster_type, default_value) if root_manager_key: experimental_arg_warning("root_manager_key", "InputDefinition") self._root_manager_key = check.opt_str_param(root_manager_key, "root_manager_key") if metadata: experimental_arg_warning("metadata", "InputDefinition") self._metadata = check.opt_dict_param(metadata, "metadata", key_type=str)
def __new__( cls, value: Any, mapping_key: str, output_name: Optional[str] = DEFAULT_OUTPUT, metadata_entries: Optional[List[Union[PartitionMetadataEntry, EventMetadataEntry]]] = None, metadata: Optional[Dict[str, ParseableMetadataEntryData]] = None, ): if metadata_entries: experimental_arg_warning("metadata_entries", "DynamicOutput.__new__") elif metadata: experimental_arg_warning("metadata", "DynamicOutput.__new__") metadata = check.opt_dict_param(metadata, "metadata", key_type=str) metadata_entries = check.opt_list_param(metadata_entries, "metadata_entries", of_type=EventMetadataEntry) return super(DynamicOutput, cls).__new__( cls, value=value, mapping_key=check_valid_name( check.str_param(mapping_key, "mapping_key")), output_name=check.str_param(output_name, "output_name"), metadata_entries=parse_metadata(metadata, metadata_entries), )
def __init__( self, name, input_defs, compute_fn, output_defs, config_schema=None, description=None, tags=None, required_resource_keys=None, positional_inputs=None, version=None, ): self._compute_fn = check.callable_param(compute_fn, "compute_fn") self._config_schema = convert_user_facing_definition_config_schema( config_schema) self._required_resource_keys = frozenset( check.opt_set_param(required_resource_keys, "required_resource_keys", of_type=str)) self._version = check.opt_str_param(version, "version") if version: experimental_arg_warning("version", "SolidDefinition.__init__") super(SolidDefinition, self).__init__( name=name, input_defs=check.list_param(input_defs, "input_defs", InputDefinition), output_defs=check.list_param(output_defs, "output_defs", OutputDefinition), description=description, tags=check.opt_dict_param(tags, "tags", key_type=str), positional_inputs=positional_inputs, )
def __new__( cls, dagster_type: Union[Type, DagsterType] = NoValueSentinel, description: Optional[str] = None, is_required: bool = True, io_manager_key: Optional[str] = None, metadata: Optional[MetadataUserInput] = None, asset_key: Optional[AssetKey] = None, asset_partitions: Optional[Union[AbstractSet[str], Callable[["OutputContext"], AbstractSet[str]]]] = None, asset_partitions_def: Optional["PartitionsDefinition"] = None, # make sure new parameters are updated in combine_with_inferred below ): if asset_partitions_def: experimental_arg_warning("asset_partitions_definition", "Out.__new__") return super(Out, cls).__new__( cls, dagster_type=NoValueSentinel if dagster_type is NoValueSentinel else resolve_dagster_type(dagster_type), description=description, is_required=check.bool_param(is_required, "is_required"), io_manager_key=check.opt_str_param(io_manager_key, "io_manager_key", default="io_manager"), metadata=metadata, asset_key=asset_key, asset_partitions=asset_partitions, asset_partitions_def=asset_partitions_def, )
def __new__( cls, value: Any, output_name: Optional[str] = DEFAULT_OUTPUT, metadata_entries: Optional[List[Union[EventMetadataEntry, PartitionMetadataEntry]]] = None, metadata: Optional[Dict[str, ParseableMetadataEntryData]] = None, ): if metadata_entries: experimental_arg_warning("metadata_entries", "Output.__new__") elif metadata: experimental_arg_warning("metadata", "Output.__new__") metadata = check.opt_dict_param(metadata, "metadata", key_type=str) metadata_entries = check.opt_list_param( metadata_entries, "metadata_entries", of_type=(EventMetadataEntry, PartitionMetadataEntry), ) return super(Output, cls).__new__( cls, value, check.str_param(output_name, "output_name"), parse_metadata(metadata, metadata_entries), )
def __new__(cls, value, output_name=DEFAULT_OUTPUT, address=None): if address: experimental_arg_warning("address", "Output.__new__") return super(Output, cls).__new__( cls, value, check.str_param(output_name, "output_name"), check.opt_str_param(address, "address"), )
def __new__(cls, value, output_name=DEFAULT_OUTPUT, metadata_entries=None, metadata=None): if metadata_entries: experimental_arg_warning("metadata_entries", "Output.__new__") elif metadata: experimental_arg_warning("metadata", "Output.__new__") return super(Output, cls).__new__( cls, value, check.str_param(output_name, "output_name"), parse_metadata(metadata, metadata_entries), )
def __init__( self, dagster_type=None, name=None, description=None, is_required=None, io_manager_key=None, metadata=None, asset_key=None, asset_partitions=None, # make sure new parameters are updated in combine_with_inferred below ): self._name = ( check_valid_name(check.opt_str_param(name, "name", DEFAULT_OUTPUT)) if name is not NoNameSentinel else None ) self._type_not_set = dagster_type is None self._dagster_type = resolve_dagster_type(dagster_type) self._description = check.opt_str_param(description, "description") self._is_required = check.opt_bool_param(is_required, "is_required", default=True) self._manager_key = check.opt_str_param( io_manager_key, "io_manager_key", default="io_manager" ) self._metadata = metadata if asset_key: experimental_arg_warning("asset_key", "OutputDefinition.__init__") if callable(asset_key): self._asset_key_fn = asset_key elif asset_key is not None: asset_key = check.opt_inst_param(asset_key, "asset_key", AssetKey) self._asset_key_fn = lambda _: asset_key else: self._asset_key_fn = None if asset_partitions: experimental_arg_warning("asset_partitions", "OutputDefinition.__init__") check.param_invariant( asset_key is not None, "asset_partitions", 'Cannot specify "asset_partitions" argument without also specifying "asset_key"', ) if callable(asset_partitions): self._asset_partitions_fn = asset_partitions elif asset_partitions is not None: asset_partitions = check.opt_set_param(asset_partitions, "asset_partitions", str) self._asset_partitions_fn = lambda _: asset_partitions else: self._asset_partitions_fn = None
def __new__(cls, value, output_name=DEFAULT_OUTPUT, metadata_entries=None): if metadata_entries: experimental_arg_warning("metadata_entries", "Output.__new__") return super(Output, cls).__new__( cls, value, check.str_param(output_name, "output_name"), check.opt_list_param( metadata_entries, "metadata_entries", (EventMetadataEntry, PartitionMetadataEntry), ), )
def __init__( self, name: str, input_defs: Sequence[InputDefinition], compute_fn: Union[Callable[..., Any], "DecoratedSolidFunction"], output_defs: Sequence[OutputDefinition], config_schema: Optional[Union[Dict[str, Any], IDefinitionConfigSchema]] = None, description: Optional[str] = None, tags: Optional[Dict[str, str]] = None, required_resource_keys: Optional[Union[Set[str], FrozenSet[str]]] = None, version: Optional[str] = None, retry_policy: Optional[RetryPolicy] = None, ): from .decorators.solid import DecoratedSolidFunction if isinstance(compute_fn, DecoratedSolidFunction): self._compute_fn: Union[Callable[..., Any], DecoratedSolidFunction] = compute_fn else: compute_fn = cast(Callable[..., Any], compute_fn) self._compute_fn = check.callable_param(compute_fn, "compute_fn") self._config_schema = convert_user_facing_definition_config_schema( config_schema) self._required_resource_keys = frozenset( check.opt_set_param(required_resource_keys, "required_resource_keys", of_type=str)) self._version = check.opt_str_param(version, "version") if version: experimental_arg_warning("version", "SolidDefinition.__init__") self._retry_policy = check.opt_inst_param(retry_policy, "retry_policy", RetryPolicy) positional_inputs = (self._compute_fn.positional_inputs() if isinstance(self._compute_fn, DecoratedSolidFunction) else None) super(SolidDefinition, self).__init__( name=name, input_defs=check.list_param(input_defs, "input_defs", InputDefinition), output_defs=check.list_param(output_defs, "output_defs", OutputDefinition), description=description, tags=check.opt_dict_param(tags, "tags", key_type=str), positional_inputs=positional_inputs, )
def __new__( cls, value, mapping_key, output_name=DEFAULT_OUTPUT, metadata_entries=None, metadata=None ): if metadata_entries: experimental_arg_warning("metadata_entries", "DynamicOutput.__new__") elif metadata: experimental_arg_warning("metadata", "DynamicOutput.__new__") return super(DynamicOutput, cls).__new__( cls, value, check_valid_name(check.str_param(mapping_key, "mapping_key")), check.str_param(output_name, "output_name"), parse_metadata(metadata, metadata_entries), )
def __init__( self, resource_fn: Callable[["InitResourceContext"], Any], config_schema: Optional[Union[Any, IDefinitionConfigSchema]] = None, description: Optional[str] = None, required_resource_keys: Optional[AbstractSet[str]] = None, version: Optional[str] = None, ): self._resource_fn = check.callable_param(resource_fn, "resource_fn") self._config_schema = convert_user_facing_definition_config_schema( config_schema) self._description = check.opt_str_param(description, "description") self._required_resource_keys = check.opt_set_param( required_resource_keys, "required_resource_keys") self._version = check.opt_str_param(version, "version") if version: experimental_arg_warning("version", "ResourceDefinition.__init__")
def __init__( self, resource_fn=None, config_schema=None, description=None, required_resource_keys=None, version=None, ): self._resource_fn = check.opt_callable_param(resource_fn, "resource_fn") self._config_schema = convert_user_facing_definition_config_schema(config_schema) self._description = check.opt_str_param(description, "description") self._required_resource_keys = check.opt_set_param( required_resource_keys, "required_resource_keys" ) self._version = check.opt_str_param(version, "version") if version: experimental_arg_warning("version", "ResourceDefinition.__init__")
def __init__( self, name: str, input_defs: List[InputDefinition], compute_fn: Callable[..., Any], output_defs: List[OutputDefinition], config_schema: Optional[Union[Dict[str, Any], IDefinitionConfigSchema]] = None, description: Optional[str] = None, tags: Optional[Dict[str, str]] = None, required_resource_keys: Optional[Union[Set[str], FrozenSet[str]]] = None, positional_inputs: Optional[List[str]] = None, version: Optional[str] = None, context_arg_provided: Optional[bool] = True, retry_policy: Optional[RetryPolicy] = None, ): self._compute_fn = check.callable_param(compute_fn, "compute_fn") self._config_schema = convert_user_facing_definition_config_schema( config_schema) self._required_resource_keys = frozenset( check.opt_set_param(required_resource_keys, "required_resource_keys", of_type=str)) self._version = check.opt_str_param(version, "version") if version: experimental_arg_warning("version", "SolidDefinition.__init__") self._retry_policy = check.opt_inst_param(retry_policy, "retry_policy", RetryPolicy) self._context_arg_provided = check.bool_param(context_arg_provided, "context_arg_provided") super(SolidDefinition, self).__init__( name=name, input_defs=check.list_param(input_defs, "input_defs", InputDefinition), output_defs=check.list_param(output_defs, "output_defs", OutputDefinition), description=description, tags=check.opt_dict_param(tags, "tags", key_type=str), positional_inputs=positional_inputs, )
def __init__( self, name: str, evaluation_fn: Callable[ ["SensorEvaluationContext"], Union[Generator[Union[RunRequest, SkipReason], None, None], RunRequest, SkipReason], ], pipeline_name: Optional[str] = None, solid_selection: Optional[List[Any]] = None, mode: Optional[str] = None, minimum_interval_seconds: Optional[int] = None, description: Optional[str] = None, job: Optional[Union[GraphDefinition, PipelineDefinition]] = None, decorated_fn: Optional[ Callable[ ["SensorEvaluationContext"], Union[Generator[Union[RunRequest, SkipReason], None, None], RunRequest, SkipReason], ] ] = None, ): self._name = check_valid_name(name) if pipeline_name is None and job is None: self._target: Optional[Union[DirectTarget, RepoRelativeTarget]] = None elif job is not None: experimental_arg_warning("target", "SensorDefinition.__init__") self._target = DirectTarget(job) else: self._target = RepoRelativeTarget( pipeline_name=check.str_param(pipeline_name, "pipeline_name"), mode=check.opt_str_param(mode, "mode") or DEFAULT_MODE_NAME, solid_selection=check.opt_nullable_list_param( solid_selection, "solid_selection", of_type=str ), ) self._description = check.opt_str_param(description, "description") self._evaluation_fn = check.callable_param(evaluation_fn, "evaluation_fn") self._decorated_fn = check.opt_callable_param(decorated_fn, "decorated_fn") self._min_interval = check.opt_int_param( minimum_interval_seconds, "minimum_interval_seconds", DEFAULT_SENSOR_DAEMON_INTERVAL )
def __init__( self, name, dagster_type=None, description=None, default_value=_NoValueSentinel, root_manager_key=None, metadata=None, asset_key=None, asset_partitions=None, ): self._name = check_valid_name(name) self._dagster_type = check.inst(resolve_dagster_type(dagster_type), DagsterType) self._description = check.opt_str_param(description, "description") self._default_value = _check_default_value(self._name, self._dagster_type, default_value) if root_manager_key: experimental_arg_warning("root_manager_key", "InputDefinition.__init__") self._root_manager_key = check.opt_str_param(root_manager_key, "root_manager_key") if metadata: experimental_arg_warning("metadata", "InputDefinition.__init__") self._metadata = check.opt_dict_param(metadata, "metadata", key_type=str) if asset_key: experimental_arg_warning("asset_key", "InputDefinition.__init__") self._is_asset = asset_key is not None if callable(asset_key): self._asset_key_fn = asset_key else: asset_key = check.opt_inst_param(asset_key, "asset_key", AssetKey) self._asset_key_fn = lambda _: asset_key if asset_partitions: experimental_arg_warning("asset_partitions", "InputDefinition.__init__") check.param_invariant( asset_key is not None, "asset_partitions", 'Cannot specify "asset_partitions" argument without also specifying "asset_key"', ) if callable(asset_partitions): self._asset_partitions_fn = asset_partitions else: asset_partitions = check.opt_set_param(asset_partitions, "asset_partitions", str) self._asset_partitions_fn = lambda _: asset_partitions
def __init__( self, name=None, dagster_type=None, description=None, default_value=NoValueSentinel, root_manager_key=None, metadata=None, asset_key=None, asset_partitions=None, # when adding new params, make sure to update combine_with_inferred below ): self._name = check_valid_name(name) if name else None self._type_not_set = dagster_type is None self._dagster_type = check.inst(resolve_dagster_type(dagster_type), DagsterType) self._description = check.opt_str_param(description, "description") self._default_value = _check_default_value(self._name, self._dagster_type, default_value) if root_manager_key: experimental_arg_warning("root_manager_key", "InputDefinition.__init__") self._root_manager_key = check.opt_str_param(root_manager_key, "root_manager_key") self._metadata = check.opt_dict_param(metadata, "metadata", key_type=str) if asset_key: experimental_arg_warning("asset_key", "InputDefinition.__init__") if not callable(asset_key): check.opt_inst_param(asset_key, "asset_key", AssetKey) self._asset_key = asset_key if asset_partitions: experimental_arg_warning("asset_partitions", "InputDefinition.__init__") check.param_invariant( asset_key is not None, "asset_partitions", 'Cannot specify "asset_partitions" argument without also specifying "asset_key"', ) if callable(asset_partitions): self._asset_partitions_fn = asset_partitions elif asset_partitions is not None: asset_partitions = check.opt_set_param(asset_partitions, "asset_partitions", str) self._asset_partitions_fn = lambda _: asset_partitions else: self._asset_partitions_fn = None
def __init__( self, dagster_type=None, name=None, description=None, is_required=None, io_manager_key=None, metadata=None, ): self._name = check_valid_name( check.opt_str_param(name, "name", DEFAULT_OUTPUT)) self._dagster_type = resolve_dagster_type(dagster_type) self._description = check.opt_str_param(description, "description") self._is_required = check.opt_bool_param(is_required, "is_required", default=True) self._manager_key = check.opt_str_param(io_manager_key, "io_manager_key", default="io_manager") if metadata: experimental_arg_warning("metadata", "OutputDefinition") self._metadata = metadata
def __init__( self, config_type, func, required_resource_keys, loader_version=None, external_version_fn=None, ): self._config_type = check.inst_param(config_type, "config_type", ConfigType) self._func = check.callable_param(func, "func") self._required_resource_keys = check.opt_set_param( required_resource_keys, "required_resource_keys", of_type=str ) self._loader_version = check.opt_str_param(loader_version, "loader_version") if self._loader_version: experimental_arg_warning("loader_version", "DagsterTypeLoaderFromDecorator.__init__") self._external_version_fn = check.opt_callable_param( external_version_fn, "external_version_fn" ) if self._external_version_fn: experimental_arg_warning( "external_version_fn", "DagsterTypeLoaderFromDecorator.__init__" )
def __init__( self, dagster_type=None, name=None, description=None, is_required=None, io_manager_key=None, metadata=None, asset_key=None, asset_partitions=None, ): self._name = check_valid_name( check.opt_str_param(name, "name", DEFAULT_OUTPUT)) self._dagster_type = resolve_dagster_type(dagster_type) self._description = check.opt_str_param(description, "description") self._is_required = check.opt_bool_param(is_required, "is_required", default=True) self._manager_key = check.opt_str_param(io_manager_key, "io_manager_key", default="io_manager") if metadata: experimental_arg_warning("metadata", "OutputDefinition.__init__") self._metadata = metadata if asset_key: experimental_arg_warning("asset_key", "OutputDefinition.__init__") self._is_asset = asset_key is not None if callable(asset_key): self._asset_key_fn = asset_key else: asset_key = check.opt_inst_param(asset_key, "asset_key", AssetKey) self._asset_key_fn = lambda _: asset_key if asset_partitions: experimental_arg_warning("asset_partitions", "OutputDefinition.__init__") check.param_invariant( asset_key is not None, "asset_partitions", 'Cannot specify "asset_partitions" argument without also specifying "asset_key"', ) if callable(asset_partitions): self._asset_partitions_fn = asset_partitions else: asset_partitions = check.opt_set_param(asset_partitions, "asset_partitions", str) self._asset_partitions_fn = lambda _: asset_partitions
def __init__( self, dagster_type=None, name=None, description=None, is_required=None, io_manager_key=None, metadata=None, asset_key=None, asset_partitions=None, asset_partitions_def=None # make sure new parameters are updated in combine_with_inferred below ): from dagster.core.definitions.partition import PartitionsDefinition self._name = (check_valid_name( check.opt_str_param(name, "name", DEFAULT_OUTPUT)) if name is not NoNameSentinel else None) self._type_not_set = dagster_type is None self._dagster_type = resolve_dagster_type(dagster_type) self._description = check.opt_str_param(description, "description") self._is_required = check.opt_bool_param(is_required, "is_required", default=True) self._manager_key = check.opt_str_param(io_manager_key, "io_manager_key", default="io_manager") self._metadata = check.opt_dict_param(metadata, "metadata", key_type=str) self._metadata_entries = check.is_list( normalize_metadata(self._metadata, [], allow_invalid=True), MetadataEntry) if asset_key: experimental_arg_warning("asset_key", "OutputDefinition.__init__") if callable(asset_key): warnings.warn( "Passing a function as the `asset_key` argument to `Out` or `OutputDefinition` is " "deprecated behavior and will be removed in version 0.15.0.") else: check.opt_inst_param(asset_key, "asset_key", AssetKey) self._asset_key = asset_key if asset_partitions: experimental_arg_warning("asset_partitions", "OutputDefinition.__init__") check.param_invariant( asset_key is not None, "asset_partitions", 'Cannot specify "asset_partitions" argument without also specifying "asset_key"', ) if callable(asset_partitions): self._asset_partitions_fn = asset_partitions elif asset_partitions is not None: asset_partitions = check.opt_set_param(asset_partitions, "asset_partitions", str) self._asset_partitions_fn = lambda _: asset_partitions else: self._asset_partitions_fn = None if asset_partitions_def: experimental_arg_warning("asset_partitions_def", "OutputDefinition.__init__") self._asset_partitions_def = check.opt_inst_param( asset_partitions_def, "asset_partition_def", PartitionsDefinition)
def __init__( self, dagster_type=None, name: Optional[str] = None, description: Optional[str] = None, is_required: bool = True, io_manager_key: Optional[str] = None, metadata: Optional[MetadataUserInput] = None, asset_key: Optional[Union[AssetKey, DynamicAssetKey]] = None, asset_partitions: Optional[Union[AbstractSet[str], Callable[["OutputContext"], AbstractSet[str]]]] = None, asset_partitions_def: Optional["PartitionsDefinition"] = None # make sure new parameters are updated in combine_with_inferred below ): from dagster.core.definitions.partition import PartitionsDefinition self._name = check_valid_name( check.opt_str_param(name, "name", DEFAULT_OUTPUT)) self._type_not_set = dagster_type is None self._dagster_type = resolve_dagster_type(dagster_type) self._description = check.opt_str_param(description, "description") self._is_required = check.bool_param(is_required, "is_required") self._io_manager_key = check.opt_str_param( io_manager_key, "io_manager_key", default="io_manager", ) self._metadata = check.opt_dict_param(metadata, "metadata", key_type=str) self._metadata_entries = check.is_list( normalize_metadata(self._metadata, [], allow_invalid=True), MetadataEntry) if asset_key: experimental_arg_warning("asset_key", "OutputDefinition.__init__") if callable(asset_key): warnings.warn( "Passing a function as the `asset_key` argument to `Out` or `OutputDefinition` is " "deprecated behavior and will be removed in version 0.15.0.") else: check.opt_inst_param(asset_key, "asset_key", AssetKey) self._asset_key = asset_key if asset_partitions: experimental_arg_warning("asset_partitions", "OutputDefinition.__init__") check.param_invariant( asset_key is not None, "asset_partitions", 'Cannot specify "asset_partitions" argument without also specifying "asset_key"', ) self._asset_partitions_fn: Optional[Callable[["OutputContext"], AbstractSet[str]]] if callable(asset_partitions): self._asset_partitions_fn = asset_partitions elif asset_partitions is not None: asset_partitions = check.opt_set_param(asset_partitions, "asset_partitions", str) def _fn(_context: "OutputContext") -> AbstractSet[str]: return cast(AbstractSet[str], asset_partitions) # mypy bug? self._asset_partitions_fn = _fn else: self._asset_partitions_fn = None if asset_partitions_def: experimental_arg_warning("asset_partitions_def", "OutputDefinition.__init__") self._asset_partitions_def = check.opt_inst_param( asset_partitions_def, "asset_partition_def", PartitionsDefinition)
def __init__( self, solid_defs, name=None, description=None, dependencies=None, mode_defs=None, preset_defs=None, tags=None, hook_defs=None, input_mappings=None, output_mappings=None, config_mapping=None, positional_inputs=None, _parent_pipeline_def=None, # https://github.com/dagster-io/dagster/issues/2115 ): if not name: warnings.warn( "Pipeline must have a name. Names will be required starting in 0.10.0 or later." ) name = _anonymous_pipeline_name() # For these warnings they check truthiness because they get changed to [] higher # in the stack for the decorator case if input_mappings: experimental_arg_warning("input_mappings", "PipelineDefinition") if output_mappings: experimental_arg_warning("output_mappings", "PipelineDefinition") if config_mapping is not None: experimental_arg_warning("config_mapping", "PipelineDefinition") if positional_inputs: experimental_arg_warning("positional_inputs", "PipelineDefinition") super(PipelineDefinition, self).__init__( name=name, description=description, dependencies=dependencies, node_defs=solid_defs, tags=check.opt_dict_param(tags, "tags", key_type=str), positional_inputs=positional_inputs, input_mappings=input_mappings, output_mappings=output_mappings, config_mapping=config_mapping, ) self._current_level_node_defs = solid_defs self._tags = validate_tags(tags) mode_definitions = check.opt_list_param(mode_defs, "mode_defs", of_type=ModeDefinition) if not mode_definitions: mode_definitions = [ModeDefinition()] self._mode_definitions = mode_definitions seen_modes = set() for mode_def in mode_definitions: if mode_def.name in seen_modes: raise DagsterInvalidDefinitionError(( 'Two modes seen with the name "{mode_name}" in "{pipeline_name}". ' "Modes must have unique names.").format( mode_name=mode_def.name, pipeline_name=self._name)) seen_modes.add(mode_def.name) self._hook_defs = check.opt_set_param(hook_defs, "hook_defs", of_type=HookDefinition) self._preset_defs = check.opt_list_param(preset_defs, "preset_defs", PresetDefinition) self._preset_dict = {} for preset in self._preset_defs: if preset.name in self._preset_dict: raise DagsterInvalidDefinitionError(( 'Two PresetDefinitions seen with the name "{name}" in "{pipeline_name}". ' "PresetDefinitions must have unique names.").format( name=preset.name, pipeline_name=self._name)) if preset.mode not in seen_modes: raise DagsterInvalidDefinitionError( ('PresetDefinition "{name}" in "{pipeline_name}" ' 'references mode "{mode}" which is not defined.').format( name=preset.name, pipeline_name=self._name, mode=preset.mode)) self._preset_dict[preset.name] = preset # Validate solid resource dependencies _validate_resource_dependencies( self._mode_definitions, self._current_level_node_defs, self._dagster_type_dict, self._solid_dict, self._hook_defs, ) # Validate unsatisfied inputs can be materialized from config _validate_inputs(self._dependency_structure, self._solid_dict, self._mode_definitions) # Recursively explore all nodes in the this pipeline self._all_node_defs = _build_all_node_defs( self._current_level_node_defs) self._parent_pipeline_def = check.opt_inst_param( _parent_pipeline_def, "_parent_pipeline_def", PipelineDefinition) self._cached_run_config_schemas = {} self._cached_external_pipeline = None
def pipeline( name: Union[Callable[..., Any], Optional[str]] = None, description: Optional[str] = None, mode_defs: Optional[List[ModeDefinition]] = None, preset_defs: Optional[List[PresetDefinition]] = None, tags: Optional[Dict[str, Any]] = None, hook_defs: Optional[Set[HookDefinition]] = None, input_defs: Optional[List[InputDefinition]] = None, output_defs: Optional[List[OutputDefinition]] = None, config_schema: Optional[Dict[str, Any]] = None, config_fn: Optional[Callable[[Dict[str, Any]], Dict[str, Any]]] = None, ) -> Union[PipelineDefinition, _Pipeline]: """Create a pipeline with the specified parameters from the decorated composition function. Using this decorator allows you to build up the dependency graph of the pipeline by writing a function that invokes solids and passes the output to other solids. Args: name (Optional[str]): The name of the pipeline. Must be unique within any :py:class:`RepositoryDefinition` containing the pipeline. description (Optional[str]): A human-readable description of the pipeline. mode_defs (Optional[List[ModeDefinition]]): The set of modes in which this pipeline can operate. Modes are used to attach resources, custom loggers, custom system storage options, and custom executors to a pipeline. Modes can be used, e.g., to vary available resource and logging implementations between local test and production runs. preset_defs (Optional[List[PresetDefinition]]): A set of preset collections of configuration options that may be used to execute a pipeline. A preset consists of an environment dict, an optional subset of solids to execute, and a mode selection. Presets can be used to ship common combinations of options to pipeline end users in Python code, and can be selected by tools like Dagit. tags (Optional[Dict[str, Any]]): Arbitrary metadata for any execution run of the pipeline. Values that are not strings will be json encoded and must meet the criteria that `json.loads(json.dumps(value)) == value`. These tag values may be overwritten by tag values provided at invocation time. hook_defs (Optional[Set[HookDefinition]]): A set of hook definitions applied to the pipeline. When a hook is applied to a pipeline, it will be attached to all solid instances within the pipeline. Example: .. code-block:: python @solid(output_defs=[OutputDefinition(int, "two"), OutputDefinition(int, "four")]) def emit_two_four(_) -> int: yield Output(2, "two") yield Output(4, "four") @lambda_solid def add_one(num: int) -> int: return num + 1 @lambda_solid def mult_two(num: int) -> int: return num * 2 @pipeline def math_pipeline(): two, four = emit_two_four() add_one(two) mult_two(four) """ if input_defs is not None: experimental_arg_warning("input_defs", "pipeline") if output_defs is not None: experimental_arg_warning("output_defs", "pipeline") if config_schema is not None: experimental_arg_warning("config_schema", "pipeline") if config_fn is not None: experimental_arg_warning("config_fn", "pipeline") if callable(name): check.invariant(description is None) return _Pipeline()(name) return _Pipeline( name=name, mode_defs=mode_defs, preset_defs=preset_defs, description=description, tags=tags, hook_defs=hook_defs, input_defs=input_defs, output_defs=output_defs, config_schema=config_schema, config_fn=config_fn, )
def __init__( self, solid_defs: List[NodeDefinition], name: str, description: Optional[str] = None, dependencies: Optional[Dict[Union[str, SolidInvocation], Dict[str, IDependencyDefinition]]] = None, mode_defs: Optional[List[ModeDefinition]] = None, preset_defs: Optional[List[PresetDefinition]] = None, tags: Dict[str, Any] = None, hook_defs: Optional[AbstractSet[HookDefinition]] = None, input_mappings: Optional[List[InputMapping]] = None, output_mappings: Optional[List[OutputMapping]] = None, config_mapping: Optional[ConfigMapping] = None, positional_inputs: List[str] = None, _parent_pipeline_def: Optional[ "PipelineDefinition"] = None, # https://github.com/dagster-io/dagster/issues/2115 ): # For these warnings they check truthiness because they get changed to [] higher # in the stack for the decorator case if input_mappings: experimental_arg_warning("input_mappings", "PipelineDefinition") if output_mappings: experimental_arg_warning("output_mappings", "PipelineDefinition") if config_mapping is not None: experimental_arg_warning("config_mapping", "PipelineDefinition") if positional_inputs: experimental_arg_warning("positional_inputs", "PipelineDefinition") super(PipelineDefinition, self).__init__( name=name, description=description, dependencies=dependencies, node_defs=solid_defs, tags=check.opt_dict_param(tags, "tags", key_type=str), positional_inputs=positional_inputs, input_mappings=input_mappings, output_mappings=output_mappings, config_mapping=config_mapping, ) self._current_level_node_defs = solid_defs self._tags = validate_tags(tags) mode_definitions = check.opt_list_param(mode_defs, "mode_defs", of_type=ModeDefinition) if not mode_definitions: mode_definitions = [ModeDefinition()] self._mode_definitions = mode_definitions seen_modes = set() for mode_def in mode_definitions: if mode_def.name in seen_modes: raise DagsterInvalidDefinitionError(( 'Two modes seen with the name "{mode_name}" in "{pipeline_name}". ' "Modes must have unique names.").format( mode_name=mode_def.name, pipeline_name=self._name)) seen_modes.add(mode_def.name) self._hook_defs = check.opt_set_param(hook_defs, "hook_defs", of_type=HookDefinition) self._preset_defs = check.opt_list_param(preset_defs, "preset_defs", PresetDefinition) self._preset_dict: Dict[str, PresetDefinition] = {} for preset in self._preset_defs: if preset.name in self._preset_dict: raise DagsterInvalidDefinitionError(( 'Two PresetDefinitions seen with the name "{name}" in "{pipeline_name}". ' "PresetDefinitions must have unique names.").format( name=preset.name, pipeline_name=self._name)) if preset.mode not in seen_modes: raise DagsterInvalidDefinitionError( ('PresetDefinition "{name}" in "{pipeline_name}" ' 'references mode "{mode}" which is not defined.').format( name=preset.name, pipeline_name=self._name, mode=preset.mode)) self._preset_dict[preset.name] = preset self._resource_requirements = { mode_def.name: _checked_resource_reqs_for_mode( mode_def, self._current_level_node_defs, self._dagster_type_dict, self._solid_dict, self._hook_defs, self._dependency_structure, ) for mode_def in self._mode_definitions } # Recursively explore all nodes in the this pipeline self._all_node_defs = _build_all_node_defs( self._current_level_node_defs) self._parent_pipeline_def = check.opt_inst_param( _parent_pipeline_def, "_parent_pipeline_def", PipelineDefinition) self._cached_run_config_schemas: Dict[str, "RunConfigSchema"] = {} self._cached_external_pipeline = None
def stable_function(_stable_arg, _experimental_arg): experimental_arg_warning("experimental_arg", "stable_function")
def __new__( cls, name: Optional[str] = None, resource_defs: Optional[Dict[str, ResourceDefinition]] = None, logger_defs: Optional[Dict[str, LoggerDefinition]] = None, executor_defs: Optional[List[ExecutorDefinition]] = None, description: Optional[str] = None, intermediate_storage_defs: Optional[ List["IntermediateStorageDefinition"]] = None, _config_mapping: Optional[ConfigMapping] = None, _partitioned_config: Optional["PartitionedConfig"] = None, ): from dagster.core.storage.system_storage import default_intermediate_storage_defs from .intermediate_storage import IntermediateStorageDefinition from .partition import PartitionedConfig check.opt_dict_param(resource_defs, "resource_defs", key_type=str, value_type=ResourceDefinition) if resource_defs and "io_manager" in resource_defs: resource_defs_with_defaults = resource_defs else: from dagster.core.storage.mem_io_manager import mem_io_manager resource_defs_with_defaults = merge_dicts( {"io_manager": mem_io_manager}, resource_defs or {}) if _config_mapping: experimental_arg_warning("_config_mapping", "ModeDefinition.__new__") if _partitioned_config: experimental_arg_warning("_partitioned_config", "ModeDefinition.__new__") return super(ModeDefinition, cls).__new__( cls, name=check_valid_name(name) if name else DEFAULT_MODE_NAME, resource_defs=resource_defs_with_defaults, loggers=(check.opt_dict_param(logger_defs, "logger_defs", key_type=str, value_type=LoggerDefinition) or default_loggers()), intermediate_storage_defs=check.list_param( intermediate_storage_defs if intermediate_storage_defs else default_intermediate_storage_defs, "intermediate_storage_defs", of_type=IntermediateStorageDefinition, ), executor_defs=check.list_param( executor_defs if executor_defs else default_executors, "executor_defs", of_type=ExecutorDefinition, ), description=check.opt_str_param(description, "description"), config_mapping=check.opt_inst_param(_config_mapping, "_config_mapping", ConfigMapping), partitioned_config=check.opt_inst_param(_partitioned_config, "_partitioned_config", PartitionedConfig), )