def __init__(
self,
resource_fn=None,
config_schema=None,
description=None,
_configured_config_mapping_fn=None,
version=None,
):
EXPECTED_POSITIONALS = ["*"]
fn_positionals, _ = split_function_parameters(resource_fn, EXPECTED_POSITIONALS)
missing_positional = validate_decorated_fn_positionals(fn_positionals, EXPECTED_POSITIONALS)
if missing_positional:
raise DagsterInvalidDefinitionError(
"@resource '{resource_name}' decorated function does not have required "
"positional parameter '{missing_param}'. Resource functions should only have keyword "
"arguments that match input names and a first positional parameter.".format(
resource_name=resource_fn.__name__, missing_param=missing_positional
)
)
self._resource_fn = check.opt_callable_param(resource_fn, "resource_fn")
self._config_schema = check_user_facing_opt_config_param(config_schema, "config_schema")
self._description = check.opt_str_param(description, "description")
self.__configured_config_mapping_fn = check.opt_callable_param(
_configured_config_mapping_fn, "config_mapping_fn"
)
self._version = check.opt_str_param(version, "version")
if version:
experimental_arg_warning("version", "ResourceDefinition.__init__")
def __new__(
cls,
dagster_type=NoValueSentinel,
description=None,
is_required=None,
io_manager_key=None,
metadata=None,
asset_key=None,
asset_partitions=None,
asset_partitions_def=None,
# make sure new parameters are updated in combine_with_inferred below
):
if asset_partitions_def:
experimental_arg_warning("assets_definition", "Out.__new__")
return super(Out, cls).__new__(
cls,
dagster_type=dagster_type,
description=description,
is_required=is_required,
io_manager_key=io_manager_key,
metadata=metadata,
asset_key=asset_key,
asset_partitions=asset_partitions,
asset_partitions_def=asset_partitions_def,
)
def __init__(
self,
name,
dagster_type=None,
description=None,
default_value=_NoValueSentinel,
root_manager_key=None,
metadata=None,
):
""
self._name = check_valid_name(name)
self._dagster_type = check.inst(resolve_dagster_type(dagster_type),
DagsterType)
self._description = check.opt_str_param(description, "description")
self._default_value = _check_default_value(self._name,
self._dagster_type,
default_value)
if root_manager_key:
experimental_arg_warning("root_manager_key", "InputDefinition")
self._root_manager_key = check.opt_str_param(root_manager_key,
"root_manager_key")
if metadata:
experimental_arg_warning("metadata", "InputDefinition")
self._metadata = check.opt_dict_param(metadata,
"metadata",
key_type=str)
def __new__(
cls,
value: Any,
mapping_key: str,
output_name: Optional[str] = DEFAULT_OUTPUT,
metadata_entries: Optional[List[Union[PartitionMetadataEntry,
EventMetadataEntry]]] = None,
metadata: Optional[Dict[str, ParseableMetadataEntryData]] = None,
):
if metadata_entries:
experimental_arg_warning("metadata_entries",
"DynamicOutput.__new__")
elif metadata:
experimental_arg_warning("metadata", "DynamicOutput.__new__")
metadata = check.opt_dict_param(metadata, "metadata", key_type=str)
metadata_entries = check.opt_list_param(metadata_entries,
"metadata_entries",
of_type=EventMetadataEntry)
return super(DynamicOutput, cls).__new__(
cls,
value=value,
mapping_key=check_valid_name(
check.str_param(mapping_key, "mapping_key")),
output_name=check.str_param(output_name, "output_name"),
metadata_entries=parse_metadata(metadata, metadata_entries),
)
def __init__(
self,
name,
input_defs,
compute_fn,
output_defs,
config_schema=None,
description=None,
tags=None,
required_resource_keys=None,
positional_inputs=None,
version=None,
):
self._compute_fn = check.callable_param(compute_fn, "compute_fn")
self._config_schema = convert_user_facing_definition_config_schema(
config_schema)
self._required_resource_keys = frozenset(
check.opt_set_param(required_resource_keys,
"required_resource_keys",
of_type=str))
self._version = check.opt_str_param(version, "version")
if version:
experimental_arg_warning("version", "SolidDefinition.__init__")
super(SolidDefinition, self).__init__(
name=name,
input_defs=check.list_param(input_defs, "input_defs",
InputDefinition),
output_defs=check.list_param(output_defs, "output_defs",
OutputDefinition),
description=description,
tags=check.opt_dict_param(tags, "tags", key_type=str),
positional_inputs=positional_inputs,
)
def __new__(
cls,
dagster_type: Union[Type, DagsterType] = NoValueSentinel,
description: Optional[str] = None,
is_required: bool = True,
io_manager_key: Optional[str] = None,
metadata: Optional[MetadataUserInput] = None,
asset_key: Optional[AssetKey] = None,
asset_partitions: Optional[Union[AbstractSet[str],
Callable[["OutputContext"],
AbstractSet[str]]]] = None,
asset_partitions_def: Optional["PartitionsDefinition"] = None,
# make sure new parameters are updated in combine_with_inferred below
):
if asset_partitions_def:
experimental_arg_warning("asset_partitions_definition",
"Out.__new__")
return super(Out, cls).__new__(
cls,
dagster_type=NoValueSentinel if dagster_type is NoValueSentinel
else resolve_dagster_type(dagster_type),
description=description,
is_required=check.bool_param(is_required, "is_required"),
io_manager_key=check.opt_str_param(io_manager_key,
"io_manager_key",
default="io_manager"),
metadata=metadata,
asset_key=asset_key,
asset_partitions=asset_partitions,
asset_partitions_def=asset_partitions_def,
)
def __new__(
cls,
value: Any,
output_name: Optional[str] = DEFAULT_OUTPUT,
metadata_entries: Optional[List[Union[EventMetadataEntry,
PartitionMetadataEntry]]] = None,
metadata: Optional[Dict[str, ParseableMetadataEntryData]] = None,
):
if metadata_entries:
experimental_arg_warning("metadata_entries", "Output.__new__")
elif metadata:
experimental_arg_warning("metadata", "Output.__new__")
metadata = check.opt_dict_param(metadata, "metadata", key_type=str)
metadata_entries = check.opt_list_param(
metadata_entries,
"metadata_entries",
of_type=(EventMetadataEntry, PartitionMetadataEntry),
)
return super(Output, cls).__new__(
cls,
value,
check.str_param(output_name, "output_name"),
parse_metadata(metadata, metadata_entries),
)
def __new__(cls, value, output_name=DEFAULT_OUTPUT, address=None):
if address:
experimental_arg_warning("address", "Output.__new__")
return super(Output, cls).__new__(
cls,
value,
check.str_param(output_name, "output_name"),
check.opt_str_param(address, "address"),
)
def __new__(cls, value, output_name=DEFAULT_OUTPUT, metadata_entries=None, metadata=None):
if metadata_entries:
experimental_arg_warning("metadata_entries", "Output.__new__")
elif metadata:
experimental_arg_warning("metadata", "Output.__new__")
return super(Output, cls).__new__(
cls,
value,
check.str_param(output_name, "output_name"),
parse_metadata(metadata, metadata_entries),
)
def __init__(
self,
dagster_type=None,
name=None,
description=None,
is_required=None,
io_manager_key=None,
metadata=None,
asset_key=None,
asset_partitions=None,
# make sure new parameters are updated in combine_with_inferred below
):
self._name = (
check_valid_name(check.opt_str_param(name, "name", DEFAULT_OUTPUT))
if name is not NoNameSentinel
else None
)
self._type_not_set = dagster_type is None
self._dagster_type = resolve_dagster_type(dagster_type)
self._description = check.opt_str_param(description, "description")
self._is_required = check.opt_bool_param(is_required, "is_required", default=True)
self._manager_key = check.opt_str_param(
io_manager_key, "io_manager_key", default="io_manager"
)
self._metadata = metadata
if asset_key:
experimental_arg_warning("asset_key", "OutputDefinition.__init__")
if callable(asset_key):
self._asset_key_fn = asset_key
elif asset_key is not None:
asset_key = check.opt_inst_param(asset_key, "asset_key", AssetKey)
self._asset_key_fn = lambda _: asset_key
else:
self._asset_key_fn = None
if asset_partitions:
experimental_arg_warning("asset_partitions", "OutputDefinition.__init__")
check.param_invariant(
asset_key is not None,
"asset_partitions",
'Cannot specify "asset_partitions" argument without also specifying "asset_key"',
)
if callable(asset_partitions):
self._asset_partitions_fn = asset_partitions
elif asset_partitions is not None:
asset_partitions = check.opt_set_param(asset_partitions, "asset_partitions", str)
self._asset_partitions_fn = lambda _: asset_partitions
else:
self._asset_partitions_fn = None
def __new__(cls, value, output_name=DEFAULT_OUTPUT, metadata_entries=None):
if metadata_entries:
experimental_arg_warning("metadata_entries", "Output.__new__")
return super(Output, cls).__new__(
cls,
value,
check.str_param(output_name, "output_name"),
check.opt_list_param(
metadata_entries,
"metadata_entries",
(EventMetadataEntry, PartitionMetadataEntry),
),
)
def __init__(
self,
name: str,
input_defs: Sequence[InputDefinition],
compute_fn: Union[Callable[..., Any], "DecoratedSolidFunction"],
output_defs: Sequence[OutputDefinition],
config_schema: Optional[Union[Dict[str, Any],
IDefinitionConfigSchema]] = None,
description: Optional[str] = None,
tags: Optional[Dict[str, str]] = None,
required_resource_keys: Optional[Union[Set[str],
FrozenSet[str]]] = None,
version: Optional[str] = None,
retry_policy: Optional[RetryPolicy] = None,
):
from .decorators.solid import DecoratedSolidFunction
if isinstance(compute_fn, DecoratedSolidFunction):
self._compute_fn: Union[Callable[..., Any],
DecoratedSolidFunction] = compute_fn
else:
compute_fn = cast(Callable[..., Any], compute_fn)
self._compute_fn = check.callable_param(compute_fn, "compute_fn")
self._config_schema = convert_user_facing_definition_config_schema(
config_schema)
self._required_resource_keys = frozenset(
check.opt_set_param(required_resource_keys,
"required_resource_keys",
of_type=str))
self._version = check.opt_str_param(version, "version")
if version:
experimental_arg_warning("version", "SolidDefinition.__init__")
self._retry_policy = check.opt_inst_param(retry_policy, "retry_policy",
RetryPolicy)
positional_inputs = (self._compute_fn.positional_inputs()
if isinstance(self._compute_fn,
DecoratedSolidFunction) else None)
super(SolidDefinition, self).__init__(
name=name,
input_defs=check.list_param(input_defs, "input_defs",
InputDefinition),
output_defs=check.list_param(output_defs, "output_defs",
OutputDefinition),
description=description,
tags=check.opt_dict_param(tags, "tags", key_type=str),
positional_inputs=positional_inputs,
)
def __new__(
cls, value, mapping_key, output_name=DEFAULT_OUTPUT, metadata_entries=None, metadata=None
):
if metadata_entries:
experimental_arg_warning("metadata_entries", "DynamicOutput.__new__")
elif metadata:
experimental_arg_warning("metadata", "DynamicOutput.__new__")
return super(DynamicOutput, cls).__new__(
cls,
value,
check_valid_name(check.str_param(mapping_key, "mapping_key")),
check.str_param(output_name, "output_name"),
parse_metadata(metadata, metadata_entries),
)
def __init__(
self,
resource_fn: Callable[["InitResourceContext"], Any],
config_schema: Optional[Union[Any, IDefinitionConfigSchema]] = None,
description: Optional[str] = None,
required_resource_keys: Optional[AbstractSet[str]] = None,
version: Optional[str] = None,
):
self._resource_fn = check.callable_param(resource_fn, "resource_fn")
self._config_schema = convert_user_facing_definition_config_schema(
config_schema)
self._description = check.opt_str_param(description, "description")
self._required_resource_keys = check.opt_set_param(
required_resource_keys, "required_resource_keys")
self._version = check.opt_str_param(version, "version")
if version:
experimental_arg_warning("version", "ResourceDefinition.__init__")
def __init__(
self,
resource_fn=None,
config_schema=None,
description=None,
required_resource_keys=None,
version=None,
):
self._resource_fn = check.opt_callable_param(resource_fn, "resource_fn")
self._config_schema = convert_user_facing_definition_config_schema(config_schema)
self._description = check.opt_str_param(description, "description")
self._required_resource_keys = check.opt_set_param(
required_resource_keys, "required_resource_keys"
)
self._version = check.opt_str_param(version, "version")
if version:
experimental_arg_warning("version", "ResourceDefinition.__init__")
def __init__(
self,
name: str,
input_defs: List[InputDefinition],
compute_fn: Callable[..., Any],
output_defs: List[OutputDefinition],
config_schema: Optional[Union[Dict[str, Any],
IDefinitionConfigSchema]] = None,
description: Optional[str] = None,
tags: Optional[Dict[str, str]] = None,
required_resource_keys: Optional[Union[Set[str],
FrozenSet[str]]] = None,
positional_inputs: Optional[List[str]] = None,
version: Optional[str] = None,
context_arg_provided: Optional[bool] = True,
retry_policy: Optional[RetryPolicy] = None,
):
self._compute_fn = check.callable_param(compute_fn, "compute_fn")
self._config_schema = convert_user_facing_definition_config_schema(
config_schema)
self._required_resource_keys = frozenset(
check.opt_set_param(required_resource_keys,
"required_resource_keys",
of_type=str))
self._version = check.opt_str_param(version, "version")
if version:
experimental_arg_warning("version", "SolidDefinition.__init__")
self._retry_policy = check.opt_inst_param(retry_policy, "retry_policy",
RetryPolicy)
self._context_arg_provided = check.bool_param(context_arg_provided,
"context_arg_provided")
super(SolidDefinition, self).__init__(
name=name,
input_defs=check.list_param(input_defs, "input_defs",
InputDefinition),
output_defs=check.list_param(output_defs, "output_defs",
OutputDefinition),
description=description,
tags=check.opt_dict_param(tags, "tags", key_type=str),
positional_inputs=positional_inputs,
)
def __init__(
self,
name: str,
evaluation_fn: Callable[
["SensorEvaluationContext"],
Union[Generator[Union[RunRequest, SkipReason], None, None], RunRequest, SkipReason],
],
pipeline_name: Optional[str] = None,
solid_selection: Optional[List[Any]] = None,
mode: Optional[str] = None,
minimum_interval_seconds: Optional[int] = None,
description: Optional[str] = None,
job: Optional[Union[GraphDefinition, PipelineDefinition]] = None,
decorated_fn: Optional[
Callable[
["SensorEvaluationContext"],
Union[Generator[Union[RunRequest, SkipReason], None, None], RunRequest, SkipReason],
]
] = None,
):
self._name = check_valid_name(name)
if pipeline_name is None and job is None:
self._target: Optional[Union[DirectTarget, RepoRelativeTarget]] = None
elif job is not None:
experimental_arg_warning("target", "SensorDefinition.__init__")
self._target = DirectTarget(job)
else:
self._target = RepoRelativeTarget(
pipeline_name=check.str_param(pipeline_name, "pipeline_name"),
mode=check.opt_str_param(mode, "mode") or DEFAULT_MODE_NAME,
solid_selection=check.opt_nullable_list_param(
solid_selection, "solid_selection", of_type=str
),
)
self._description = check.opt_str_param(description, "description")
self._evaluation_fn = check.callable_param(evaluation_fn, "evaluation_fn")
self._decorated_fn = check.opt_callable_param(decorated_fn, "decorated_fn")
self._min_interval = check.opt_int_param(
minimum_interval_seconds, "minimum_interval_seconds", DEFAULT_SENSOR_DAEMON_INTERVAL
)
def __init__(
self,
name,
dagster_type=None,
description=None,
default_value=_NoValueSentinel,
root_manager_key=None,
metadata=None,
asset_key=None,
asset_partitions=None,
):
self._name = check_valid_name(name)
self._dagster_type = check.inst(resolve_dagster_type(dagster_type), DagsterType)
self._description = check.opt_str_param(description, "description")
self._default_value = _check_default_value(self._name, self._dagster_type, default_value)
if root_manager_key:
experimental_arg_warning("root_manager_key", "InputDefinition.__init__")
self._root_manager_key = check.opt_str_param(root_manager_key, "root_manager_key")
if metadata:
experimental_arg_warning("metadata", "InputDefinition.__init__")
self._metadata = check.opt_dict_param(metadata, "metadata", key_type=str)
if asset_key:
experimental_arg_warning("asset_key", "InputDefinition.__init__")
self._is_asset = asset_key is not None
if callable(asset_key):
self._asset_key_fn = asset_key
else:
asset_key = check.opt_inst_param(asset_key, "asset_key", AssetKey)
self._asset_key_fn = lambda _: asset_key
if asset_partitions:
experimental_arg_warning("asset_partitions", "InputDefinition.__init__")
check.param_invariant(
asset_key is not None,
"asset_partitions",
'Cannot specify "asset_partitions" argument without also specifying "asset_key"',
)
if callable(asset_partitions):
self._asset_partitions_fn = asset_partitions
else:
asset_partitions = check.opt_set_param(asset_partitions, "asset_partitions", str)
self._asset_partitions_fn = lambda _: asset_partitions
def __init__(
self,
name=None,
dagster_type=None,
description=None,
default_value=NoValueSentinel,
root_manager_key=None,
metadata=None,
asset_key=None,
asset_partitions=None,
# when adding new params, make sure to update combine_with_inferred below
):
self._name = check_valid_name(name) if name else None
self._type_not_set = dagster_type is None
self._dagster_type = check.inst(resolve_dagster_type(dagster_type),
DagsterType)
self._description = check.opt_str_param(description, "description")
self._default_value = _check_default_value(self._name,
self._dagster_type,
default_value)
if root_manager_key:
experimental_arg_warning("root_manager_key",
"InputDefinition.__init__")
self._root_manager_key = check.opt_str_param(root_manager_key,
"root_manager_key")
self._metadata = check.opt_dict_param(metadata,
"metadata",
key_type=str)
if asset_key:
experimental_arg_warning("asset_key", "InputDefinition.__init__")
if not callable(asset_key):
check.opt_inst_param(asset_key, "asset_key", AssetKey)
self._asset_key = asset_key
if asset_partitions:
experimental_arg_warning("asset_partitions",
"InputDefinition.__init__")
check.param_invariant(
asset_key is not None,
"asset_partitions",
'Cannot specify "asset_partitions" argument without also specifying "asset_key"',
)
if callable(asset_partitions):
self._asset_partitions_fn = asset_partitions
elif asset_partitions is not None:
asset_partitions = check.opt_set_param(asset_partitions,
"asset_partitions", str)
self._asset_partitions_fn = lambda _: asset_partitions
else:
self._asset_partitions_fn = None
def __init__(
self,
dagster_type=None,
name=None,
description=None,
is_required=None,
io_manager_key=None,
metadata=None,
):
self._name = check_valid_name(
check.opt_str_param(name, "name", DEFAULT_OUTPUT))
self._dagster_type = resolve_dagster_type(dagster_type)
self._description = check.opt_str_param(description, "description")
self._is_required = check.opt_bool_param(is_required,
"is_required",
default=True)
self._manager_key = check.opt_str_param(io_manager_key,
"io_manager_key",
default="io_manager")
if metadata:
experimental_arg_warning("metadata", "OutputDefinition")
self._metadata = metadata
def __init__(
self,
config_type,
func,
required_resource_keys,
loader_version=None,
external_version_fn=None,
):
self._config_type = check.inst_param(config_type, "config_type", ConfigType)
self._func = check.callable_param(func, "func")
self._required_resource_keys = check.opt_set_param(
required_resource_keys, "required_resource_keys", of_type=str
)
self._loader_version = check.opt_str_param(loader_version, "loader_version")
if self._loader_version:
experimental_arg_warning("loader_version", "DagsterTypeLoaderFromDecorator.__init__")
self._external_version_fn = check.opt_callable_param(
external_version_fn, "external_version_fn"
)
if self._external_version_fn:
experimental_arg_warning(
"external_version_fn", "DagsterTypeLoaderFromDecorator.__init__"
)
def __init__(
self,
dagster_type=None,
name=None,
description=None,
is_required=None,
io_manager_key=None,
metadata=None,
asset_key=None,
asset_partitions=None,
):
self._name = check_valid_name(
check.opt_str_param(name, "name", DEFAULT_OUTPUT))
self._dagster_type = resolve_dagster_type(dagster_type)
self._description = check.opt_str_param(description, "description")
self._is_required = check.opt_bool_param(is_required,
"is_required",
default=True)
self._manager_key = check.opt_str_param(io_manager_key,
"io_manager_key",
default="io_manager")
if metadata:
experimental_arg_warning("metadata", "OutputDefinition.__init__")
self._metadata = metadata
if asset_key:
experimental_arg_warning("asset_key", "OutputDefinition.__init__")
self._is_asset = asset_key is not None
if callable(asset_key):
self._asset_key_fn = asset_key
else:
asset_key = check.opt_inst_param(asset_key, "asset_key", AssetKey)
self._asset_key_fn = lambda _: asset_key
if asset_partitions:
experimental_arg_warning("asset_partitions",
"OutputDefinition.__init__")
check.param_invariant(
asset_key is not None,
"asset_partitions",
'Cannot specify "asset_partitions" argument without also specifying "asset_key"',
)
if callable(asset_partitions):
self._asset_partitions_fn = asset_partitions
else:
asset_partitions = check.opt_set_param(asset_partitions,
"asset_partitions", str)
self._asset_partitions_fn = lambda _: asset_partitions
def __init__(
self,
dagster_type=None,
name=None,
description=None,
is_required=None,
io_manager_key=None,
metadata=None,
asset_key=None,
asset_partitions=None,
asset_partitions_def=None
# make sure new parameters are updated in combine_with_inferred below
):
from dagster.core.definitions.partition import PartitionsDefinition
self._name = (check_valid_name(
check.opt_str_param(name, "name", DEFAULT_OUTPUT))
if name is not NoNameSentinel else None)
self._type_not_set = dagster_type is None
self._dagster_type = resolve_dagster_type(dagster_type)
self._description = check.opt_str_param(description, "description")
self._is_required = check.opt_bool_param(is_required,
"is_required",
default=True)
self._manager_key = check.opt_str_param(io_manager_key,
"io_manager_key",
default="io_manager")
self._metadata = check.opt_dict_param(metadata,
"metadata",
key_type=str)
self._metadata_entries = check.is_list(
normalize_metadata(self._metadata, [], allow_invalid=True),
MetadataEntry)
if asset_key:
experimental_arg_warning("asset_key", "OutputDefinition.__init__")
if callable(asset_key):
warnings.warn(
"Passing a function as the `asset_key` argument to `Out` or `OutputDefinition` is "
"deprecated behavior and will be removed in version 0.15.0.")
else:
check.opt_inst_param(asset_key, "asset_key", AssetKey)
self._asset_key = asset_key
if asset_partitions:
experimental_arg_warning("asset_partitions",
"OutputDefinition.__init__")
check.param_invariant(
asset_key is not None,
"asset_partitions",
'Cannot specify "asset_partitions" argument without also specifying "asset_key"',
)
if callable(asset_partitions):
self._asset_partitions_fn = asset_partitions
elif asset_partitions is not None:
asset_partitions = check.opt_set_param(asset_partitions,
"asset_partitions", str)
self._asset_partitions_fn = lambda _: asset_partitions
else:
self._asset_partitions_fn = None
if asset_partitions_def:
experimental_arg_warning("asset_partitions_def",
"OutputDefinition.__init__")
self._asset_partitions_def = check.opt_inst_param(
asset_partitions_def, "asset_partition_def", PartitionsDefinition)
def __init__(
self,
dagster_type=None,
name: Optional[str] = None,
description: Optional[str] = None,
is_required: bool = True,
io_manager_key: Optional[str] = None,
metadata: Optional[MetadataUserInput] = None,
asset_key: Optional[Union[AssetKey, DynamicAssetKey]] = None,
asset_partitions: Optional[Union[AbstractSet[str],
Callable[["OutputContext"],
AbstractSet[str]]]] = None,
asset_partitions_def: Optional["PartitionsDefinition"] = None
# make sure new parameters are updated in combine_with_inferred below
):
from dagster.core.definitions.partition import PartitionsDefinition
self._name = check_valid_name(
check.opt_str_param(name, "name", DEFAULT_OUTPUT))
self._type_not_set = dagster_type is None
self._dagster_type = resolve_dagster_type(dagster_type)
self._description = check.opt_str_param(description, "description")
self._is_required = check.bool_param(is_required, "is_required")
self._io_manager_key = check.opt_str_param(
io_manager_key,
"io_manager_key",
default="io_manager",
)
self._metadata = check.opt_dict_param(metadata,
"metadata",
key_type=str)
self._metadata_entries = check.is_list(
normalize_metadata(self._metadata, [], allow_invalid=True),
MetadataEntry)
if asset_key:
experimental_arg_warning("asset_key", "OutputDefinition.__init__")
if callable(asset_key):
warnings.warn(
"Passing a function as the `asset_key` argument to `Out` or `OutputDefinition` is "
"deprecated behavior and will be removed in version 0.15.0.")
else:
check.opt_inst_param(asset_key, "asset_key", AssetKey)
self._asset_key = asset_key
if asset_partitions:
experimental_arg_warning("asset_partitions",
"OutputDefinition.__init__")
check.param_invariant(
asset_key is not None,
"asset_partitions",
'Cannot specify "asset_partitions" argument without also specifying "asset_key"',
)
self._asset_partitions_fn: Optional[Callable[["OutputContext"],
AbstractSet[str]]]
if callable(asset_partitions):
self._asset_partitions_fn = asset_partitions
elif asset_partitions is not None:
asset_partitions = check.opt_set_param(asset_partitions,
"asset_partitions", str)
def _fn(_context: "OutputContext") -> AbstractSet[str]:
return cast(AbstractSet[str], asset_partitions) # mypy bug?
self._asset_partitions_fn = _fn
else:
self._asset_partitions_fn = None
if asset_partitions_def:
experimental_arg_warning("asset_partitions_def",
"OutputDefinition.__init__")
self._asset_partitions_def = check.opt_inst_param(
asset_partitions_def, "asset_partition_def", PartitionsDefinition)
def __init__(
self,
solid_defs,
name=None,
description=None,
dependencies=None,
mode_defs=None,
preset_defs=None,
tags=None,
hook_defs=None,
input_mappings=None,
output_mappings=None,
config_mapping=None,
positional_inputs=None,
_parent_pipeline_def=None, # https://github.com/dagster-io/dagster/issues/2115
):
if not name:
warnings.warn(
"Pipeline must have a name. Names will be required starting in 0.10.0 or later."
)
name = _anonymous_pipeline_name()
# For these warnings they check truthiness because they get changed to [] higher
# in the stack for the decorator case
if input_mappings:
experimental_arg_warning("input_mappings", "PipelineDefinition")
if output_mappings:
experimental_arg_warning("output_mappings", "PipelineDefinition")
if config_mapping is not None:
experimental_arg_warning("config_mapping", "PipelineDefinition")
if positional_inputs:
experimental_arg_warning("positional_inputs", "PipelineDefinition")
super(PipelineDefinition, self).__init__(
name=name,
description=description,
dependencies=dependencies,
node_defs=solid_defs,
tags=check.opt_dict_param(tags, "tags", key_type=str),
positional_inputs=positional_inputs,
input_mappings=input_mappings,
output_mappings=output_mappings,
config_mapping=config_mapping,
)
self._current_level_node_defs = solid_defs
self._tags = validate_tags(tags)
mode_definitions = check.opt_list_param(mode_defs,
"mode_defs",
of_type=ModeDefinition)
if not mode_definitions:
mode_definitions = [ModeDefinition()]
self._mode_definitions = mode_definitions
seen_modes = set()
for mode_def in mode_definitions:
if mode_def.name in seen_modes:
raise DagsterInvalidDefinitionError((
'Two modes seen with the name "{mode_name}" in "{pipeline_name}". '
"Modes must have unique names.").format(
mode_name=mode_def.name, pipeline_name=self._name))
seen_modes.add(mode_def.name)
self._hook_defs = check.opt_set_param(hook_defs,
"hook_defs",
of_type=HookDefinition)
self._preset_defs = check.opt_list_param(preset_defs, "preset_defs",
PresetDefinition)
self._preset_dict = {}
for preset in self._preset_defs:
if preset.name in self._preset_dict:
raise DagsterInvalidDefinitionError((
'Two PresetDefinitions seen with the name "{name}" in "{pipeline_name}". '
"PresetDefinitions must have unique names.").format(
name=preset.name, pipeline_name=self._name))
if preset.mode not in seen_modes:
raise DagsterInvalidDefinitionError(
('PresetDefinition "{name}" in "{pipeline_name}" '
'references mode "{mode}" which is not defined.').format(
name=preset.name,
pipeline_name=self._name,
mode=preset.mode))
self._preset_dict[preset.name] = preset
# Validate solid resource dependencies
_validate_resource_dependencies(
self._mode_definitions,
self._current_level_node_defs,
self._dagster_type_dict,
self._solid_dict,
self._hook_defs,
)
# Validate unsatisfied inputs can be materialized from config
_validate_inputs(self._dependency_structure, self._solid_dict,
self._mode_definitions)
# Recursively explore all nodes in the this pipeline
self._all_node_defs = _build_all_node_defs(
self._current_level_node_defs)
self._parent_pipeline_def = check.opt_inst_param(
_parent_pipeline_def, "_parent_pipeline_def", PipelineDefinition)
self._cached_run_config_schemas = {}
self._cached_external_pipeline = None
def pipeline(
name: Union[Callable[..., Any], Optional[str]] = None,
description: Optional[str] = None,
mode_defs: Optional[List[ModeDefinition]] = None,
preset_defs: Optional[List[PresetDefinition]] = None,
tags: Optional[Dict[str, Any]] = None,
hook_defs: Optional[Set[HookDefinition]] = None,
input_defs: Optional[List[InputDefinition]] = None,
output_defs: Optional[List[OutputDefinition]] = None,
config_schema: Optional[Dict[str, Any]] = None,
config_fn: Optional[Callable[[Dict[str, Any]], Dict[str, Any]]] = None,
) -> Union[PipelineDefinition, _Pipeline]:
"""Create a pipeline with the specified parameters from the decorated composition function.
Using this decorator allows you to build up the dependency graph of the pipeline by writing a
function that invokes solids and passes the output to other solids.
Args:
name (Optional[str]): The name of the pipeline. Must be unique within any
:py:class:`RepositoryDefinition` containing the pipeline.
description (Optional[str]): A human-readable description of the pipeline.
mode_defs (Optional[List[ModeDefinition]]): The set of modes in which this pipeline can
operate. Modes are used to attach resources, custom loggers, custom system storage
options, and custom executors to a pipeline. Modes can be used, e.g., to vary
available resource and logging implementations between local test and production runs.
preset_defs (Optional[List[PresetDefinition]]): A set of preset collections of configuration
options that may be used to execute a pipeline. A preset consists of an environment
dict, an optional subset of solids to execute, and a mode selection. Presets can be used
to ship common combinations of options to pipeline end users in Python code, and can
be selected by tools like Dagit.
tags (Optional[Dict[str, Any]]): Arbitrary metadata for any execution run of the pipeline.
Values that are not strings will be json encoded and must meet the criteria that
`json.loads(json.dumps(value)) == value`. These tag values may be overwritten by tag
values provided at invocation time.
hook_defs (Optional[Set[HookDefinition]]): A set of hook definitions applied to the
pipeline. When a hook is applied to a pipeline, it will be attached to all solid
instances within the pipeline.
Example:
.. code-block:: python
@solid(output_defs=[OutputDefinition(int, "two"), OutputDefinition(int, "four")])
def emit_two_four(_) -> int:
yield Output(2, "two")
yield Output(4, "four")
@lambda_solid
def add_one(num: int) -> int:
return num + 1
@lambda_solid
def mult_two(num: int) -> int:
return num * 2
@pipeline
def math_pipeline():
two, four = emit_two_four()
add_one(two)
mult_two(four)
"""
if input_defs is not None:
experimental_arg_warning("input_defs", "pipeline")
if output_defs is not None:
experimental_arg_warning("output_defs", "pipeline")
if config_schema is not None:
experimental_arg_warning("config_schema", "pipeline")
if config_fn is not None:
experimental_arg_warning("config_fn", "pipeline")
if callable(name):
check.invariant(description is None)
return _Pipeline()(name)
return _Pipeline(
name=name,
mode_defs=mode_defs,
preset_defs=preset_defs,
description=description,
tags=tags,
hook_defs=hook_defs,
input_defs=input_defs,
output_defs=output_defs,
config_schema=config_schema,
config_fn=config_fn,
)
def __init__(
self,
solid_defs: List[NodeDefinition],
name: str,
description: Optional[str] = None,
dependencies: Optional[Dict[Union[str, SolidInvocation],
Dict[str,
IDependencyDefinition]]] = None,
mode_defs: Optional[List[ModeDefinition]] = None,
preset_defs: Optional[List[PresetDefinition]] = None,
tags: Dict[str, Any] = None,
hook_defs: Optional[AbstractSet[HookDefinition]] = None,
input_mappings: Optional[List[InputMapping]] = None,
output_mappings: Optional[List[OutputMapping]] = None,
config_mapping: Optional[ConfigMapping] = None,
positional_inputs: List[str] = None,
_parent_pipeline_def:
Optional[
"PipelineDefinition"] = None, # https://github.com/dagster-io/dagster/issues/2115
):
# For these warnings they check truthiness because they get changed to [] higher
# in the stack for the decorator case
if input_mappings:
experimental_arg_warning("input_mappings", "PipelineDefinition")
if output_mappings:
experimental_arg_warning("output_mappings", "PipelineDefinition")
if config_mapping is not None:
experimental_arg_warning("config_mapping", "PipelineDefinition")
if positional_inputs:
experimental_arg_warning("positional_inputs", "PipelineDefinition")
super(PipelineDefinition, self).__init__(
name=name,
description=description,
dependencies=dependencies,
node_defs=solid_defs,
tags=check.opt_dict_param(tags, "tags", key_type=str),
positional_inputs=positional_inputs,
input_mappings=input_mappings,
output_mappings=output_mappings,
config_mapping=config_mapping,
)
self._current_level_node_defs = solid_defs
self._tags = validate_tags(tags)
mode_definitions = check.opt_list_param(mode_defs,
"mode_defs",
of_type=ModeDefinition)
if not mode_definitions:
mode_definitions = [ModeDefinition()]
self._mode_definitions = mode_definitions
seen_modes = set()
for mode_def in mode_definitions:
if mode_def.name in seen_modes:
raise DagsterInvalidDefinitionError((
'Two modes seen with the name "{mode_name}" in "{pipeline_name}". '
"Modes must have unique names.").format(
mode_name=mode_def.name, pipeline_name=self._name))
seen_modes.add(mode_def.name)
self._hook_defs = check.opt_set_param(hook_defs,
"hook_defs",
of_type=HookDefinition)
self._preset_defs = check.opt_list_param(preset_defs, "preset_defs",
PresetDefinition)
self._preset_dict: Dict[str, PresetDefinition] = {}
for preset in self._preset_defs:
if preset.name in self._preset_dict:
raise DagsterInvalidDefinitionError((
'Two PresetDefinitions seen with the name "{name}" in "{pipeline_name}". '
"PresetDefinitions must have unique names.").format(
name=preset.name, pipeline_name=self._name))
if preset.mode not in seen_modes:
raise DagsterInvalidDefinitionError(
('PresetDefinition "{name}" in "{pipeline_name}" '
'references mode "{mode}" which is not defined.').format(
name=preset.name,
pipeline_name=self._name,
mode=preset.mode))
self._preset_dict[preset.name] = preset
self._resource_requirements = {
mode_def.name: _checked_resource_reqs_for_mode(
mode_def,
self._current_level_node_defs,
self._dagster_type_dict,
self._solid_dict,
self._hook_defs,
self._dependency_structure,
)
for mode_def in self._mode_definitions
}
# Recursively explore all nodes in the this pipeline
self._all_node_defs = _build_all_node_defs(
self._current_level_node_defs)
self._parent_pipeline_def = check.opt_inst_param(
_parent_pipeline_def, "_parent_pipeline_def", PipelineDefinition)
self._cached_run_config_schemas: Dict[str, "RunConfigSchema"] = {}
self._cached_external_pipeline = None
def stable_function(_stable_arg, _experimental_arg):
experimental_arg_warning("experimental_arg", "stable_function")
def __new__(
cls,
name: Optional[str] = None,
resource_defs: Optional[Dict[str, ResourceDefinition]] = None,
logger_defs: Optional[Dict[str, LoggerDefinition]] = None,
executor_defs: Optional[List[ExecutorDefinition]] = None,
description: Optional[str] = None,
intermediate_storage_defs: Optional[
List["IntermediateStorageDefinition"]] = None,
_config_mapping: Optional[ConfigMapping] = None,
_partitioned_config: Optional["PartitionedConfig"] = None,
):
from dagster.core.storage.system_storage import default_intermediate_storage_defs
from .intermediate_storage import IntermediateStorageDefinition
from .partition import PartitionedConfig
check.opt_dict_param(resource_defs,
"resource_defs",
key_type=str,
value_type=ResourceDefinition)
if resource_defs and "io_manager" in resource_defs:
resource_defs_with_defaults = resource_defs
else:
from dagster.core.storage.mem_io_manager import mem_io_manager
resource_defs_with_defaults = merge_dicts(
{"io_manager": mem_io_manager}, resource_defs or {})
if _config_mapping:
experimental_arg_warning("_config_mapping",
"ModeDefinition.__new__")
if _partitioned_config:
experimental_arg_warning("_partitioned_config",
"ModeDefinition.__new__")
return super(ModeDefinition, cls).__new__(
cls,
name=check_valid_name(name) if name else DEFAULT_MODE_NAME,
resource_defs=resource_defs_with_defaults,
loggers=(check.opt_dict_param(logger_defs,
"logger_defs",
key_type=str,
value_type=LoggerDefinition)
or default_loggers()),
intermediate_storage_defs=check.list_param(
intermediate_storage_defs if intermediate_storage_defs else
default_intermediate_storage_defs,
"intermediate_storage_defs",
of_type=IntermediateStorageDefinition,
),
executor_defs=check.list_param(
executor_defs if executor_defs else default_executors,
"executor_defs",
of_type=ExecutorDefinition,
),
description=check.opt_str_param(description, "description"),
config_mapping=check.opt_inst_param(_config_mapping,
"_config_mapping",
ConfigMapping),
partitioned_config=check.opt_inst_param(_partitioned_config,
"_partitioned_config",
PartitionedConfig),
)
