def __new__(
cls,
asset_key: Union[List[str], AssetKey, str],
description: Optional[str] = None,
metadata_entries: Optional[List[EventMetadataEntry]] = None,
partition: Optional[str] = None,
metadata: Optional[Dict[str, ParseableMetadataEntryData]] = None,
):
if isinstance(asset_key, AssetKey):
check.inst_param(asset_key, "asset_key", AssetKey)
elif isinstance(asset_key, str):
asset_key = AssetKey(parse_asset_key_string(asset_key))
elif isinstance(asset_key, list):
check.is_list(asset_key, of_type=str)
asset_key = AssetKey(asset_key)
else:
check.is_tuple(asset_key, of_type=str)
asset_key = AssetKey(asset_key)
metadata = check.opt_dict_param(metadata, "metadata", key_type=str)
metadata_entries = check.opt_list_param(metadata_entries,
"metadata_entries",
of_type=EventMetadataEntry)
return super(AssetObservation, cls).__new__(
cls,
asset_key=asset_key,
description=check.opt_str_param(description, "description"),
metadata_entries=cast(List[EventMetadataEntry],
parse_metadata(metadata, metadata_entries)),
partition=check.opt_str_param(partition, "partition"),
)
def __new__(
cls,
asset_key,
description=None,
metadata_entries=None,
partition=None,
tags=None,
metadata=None,
):
if isinstance(asset_key, AssetKey):
check.inst_param(asset_key, "asset_key", AssetKey)
elif isinstance(asset_key, str):
asset_key = AssetKey(parse_asset_key_string(asset_key))
elif isinstance(asset_key, list):
check.is_list(asset_key, of_type=str)
asset_key = AssetKey(asset_key)
else:
check.is_tuple(asset_key, of_type=str)
asset_key = AssetKey(asset_key)
if tags:
experimental_class_param_warning("tags", "AssetMaterialization")
return super(AssetMaterialization, cls).__new__(
cls,
asset_key=asset_key,
description=check.opt_str_param(description, "description"),
metadata_entries=parse_metadata(metadata, metadata_entries),
partition=check.opt_str_param(partition, "partition"),
tags=check.opt_dict_param(tags,
"tags",
key_type=str,
value_type=str),
)
def from_dict(cls, d: Dict[str, Any]) -> "DbtResult":
"""Constructs an instance of :class:`DbtResult <dagster_dbt.DbtResult>` from a dictionary.
Args:
d (Dict[str, Any]): A dictionary with key-values to construct a :class:`DbtResult
<dagster_dbt.DbtResult>`.
Returns:
DbtResult: An instance of :class:`DbtResult <dagster_dbt.DbtResult>`.
"""
check.list_elem(d, "logs")
logs = check.is_list(d["logs"], of_type=Dict)
check.list_elem(d, "results")
results = [
NodeResult.from_dict(d)
for d in check.is_list(d["results"], of_type=Dict)
]
generated_at = check.str_elem(d, "generated_at")
elapsed_time = check.float_elem(d, "elapsed_time")
return cls(
logs=logs,
results=results,
generated_at=generated_at,
elapsed_time=elapsed_time,
)
def __new__(cls,
asset_key,
description=None,
metadata_entries=None,
partition=None):
if isinstance(asset_key, AssetKey):
check.inst_param(asset_key, "asset_key", AssetKey)
elif isinstance(asset_key, str):
asset_key = AssetKey(parse_asset_key_string(asset_key))
elif isinstance(asset_key, list):
check.is_list(asset_key, of_type=str)
asset_key = AssetKey(asset_key)
else:
check.is_tuple(asset_key, of_type=str)
asset_key = AssetKey(asset_key)
return super(AssetMaterialization, cls).__new__(
cls,
asset_key=asset_key,
description=check.opt_str_param(description, "description"),
metadata_entries=check.opt_list_param(metadata_entries,
metadata_entries,
of_type=EventMetadataEntry),
partition=check.opt_str_param(partition, "partition"),
)
def test_is_list():
assert check.is_list([]) == []
with pytest.raises(CheckError):
check.is_list(None)
with pytest.raises(CheckError):
check.is_list('3u4')
def evaluate_tick(
self,
context: "ScheduleEvaluationContext") -> ScheduleExecutionData:
"""Evaluate schedule using the provided context.
Args:
context (ScheduleEvaluationContext): The context with which to evaluate this schedule.
Returns:
ScheduleExecutionData: Contains list of run requests, or skip message if present.
"""
from .decorators.schedule import DecoratedScheduleFunction
check.inst_param(context, "context", ScheduleEvaluationContext)
if isinstance(self._execution_fn, DecoratedScheduleFunction):
execution_fn = self._execution_fn.wrapped_fn
else:
execution_fn = cast(Callable[[ScheduleEvaluationContext], Any],
self._execution_fn)
result = list(ensure_gen(execution_fn(context)))
skip_message: Optional[str] = None
if not result or result == [None]:
run_requests = []
skip_message = "Schedule function returned an empty result"
elif len(result) == 1:
item = result[0]
check.inst(item, (SkipReason, RunRequest))
run_requests = [item] if isinstance(item, RunRequest) else []
skip_message = item.skip_message if isinstance(
item, SkipReason) else None
else:
check.is_list(result, of_type=RunRequest)
check.invariant(
not any(not request.run_key for request in result),
"Schedules that return multiple RunRequests must specify a run_key in each RunRequest",
)
run_requests = result
skip_message = None
# clone all the run requests with the required schedule tags
run_requests_with_schedule_tags = [
RunRequest(
run_key=request.run_key,
run_config=request.run_config,
tags=merge_dicts(request.tags,
PipelineRun.tags_for_schedule(self)),
) for request in run_requests
]
return ScheduleExecutionData(
run_requests=run_requests_with_schedule_tags,
skip_message=skip_message)
def get_execution_data(self, context):
check.inst_param(context, "context", SensorExecutionContext)
result = list(ensure_gen(self._evaluation_fn(context)))
if not result or result == [None]:
return []
if len(result) == 1:
return check.is_list(result, of_type=(RunRequest, SkipReason))
return check.is_list(result, of_type=RunRequest)
def test_schedule_namedtuple_job_instigator_backcompat():
src_dir = file_relative_path(__file__, "snapshot_0_13_19_instigator_named_tuples/sqlite")
with copy_directory(src_dir) as test_dir:
with DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) as instance:
states = instance.all_instigator_state()
assert len(states) == 2
check.is_list(states, of_type=InstigatorState)
for state in states:
assert state.instigator_type
assert state.instigator_data
ticks = instance.get_ticks(state.instigator_origin_id, state.selector_id)
check.is_list(ticks, of_type=InstigatorTick)
for tick in ticks:
assert tick.tick_data
assert tick.instigator_type
assert tick.instigator_name
def from_dict(cls, d: Dict[str, Any]) -> "NodeResult":
"""Constructs an instance of :class:`NodeResult <dagster_dbt.NodeResult>` from a dictionary.
Args:
d (Dict[str, Any]): A dictionary with key-values to construct a :class:`NodeResult
<dagster_dbt.NodeResult>`.
Returns:
NodeResult: An instance of :class:`NodeResult <dagster_dbt.NodeResult>`.
"""
node = check.dict_elem(d, "node")
error = check.opt_str_elem(d, "error")
execution_time = check.float_elem(d, "execution_time")
thread_id = check.opt_str_elem(d, "thread_id")
check.list_elem(d, "timing")
step_timings = [
StepTiming(
name=d["name"],
started_at=parser.isoparse(d["started_at"]),
completed_at=parser.isoparse(d["completed_at"]),
)
for d in check.is_list(d["timing"], of_type=Dict)
]
table = check.opt_dict_elem(d, "table")
return cls(
step_timings=step_timings,
node=node,
error=error,
execution_time=execution_time,
thread_id=thread_id,
table=table,
)
def inner_type_key(self) -> str:
# valid for Noneable and Array
check.invariant(self.kind == ConfigTypeKind.NONEABLE
or self.kind == ConfigTypeKind.ARRAY)
type_param_keys = check.is_list(self.type_param_keys, of_type=str)
check.invariant(len(type_param_keys) == 1)
return type_param_keys[0]
def __new__(cls,
asset_key,
description=None,
metadata_entries=None,
dagster_type=None):
if isinstance(asset_key, AssetKey):
check.inst_param(asset_key, 'asset_key', AssetKey)
elif check.is_str(asset_key):
asset_key = AssetKey(parse_asset_key_string(asset_key))
elif isinstance(asset_key, list):
check.is_list(asset_key, of_type=str)
asset_key = AssetKey(asset_key)
else:
check.is_tuple(asset_key, of_type=str)
asset_key = AssetKey(asset_key)
if dagster_type is not None:
from dagster.core.types.dagster_type import DagsterType
# importing here to resolve circularity
dagster_type = check.inst_param(dagster_type, 'dagster_type',
DagsterType)
metadata_entries = check.opt_list_param(metadata_entries,
metadata_entries,
of_type=EventMetadataEntry)
if metadata_entries is None:
metadata_entries = []
metadata_entries.append(
EventMetadataEntry.text(
(dagster_type.name if dagster_type.name else 'Any'),
'system-type-name'))
metadata_entries.append(
EventMetadataEntry.text(
(dagster_type.description
if dagster_type.description else 'Any'),
'system-type-description',
))
return super(AssetMaterialization, cls).__new__(
cls,
asset_key=asset_key,
description=check.opt_str_param(description, 'description'),
metadata_entries=check.opt_list_param(metadata_entries,
metadata_entries,
of_type=EventMetadataEntry),
)
def key_type_key(self) -> str:
"""For a type which has keys such as Map, returns the type of the key."""
# valid for Map, which has its key type as the first entry in type_param_keys
check.invariant(self.kind == ConfigTypeKind.MAP)
type_param_keys = check.is_list(self.type_param_keys, of_type=str)
check.invariant(len(type_param_keys) == 2)
return type_param_keys[0]
def _get_field(self, name: str) -> Optional["ConfigFieldSnap"]:
check.str_param(name, "name")
check.invariant(ConfigTypeKind.has_fields(self.kind))
fields = check.is_list(self.fields, of_type=ConfigFieldSnap)
for f in fields:
if f.name == name:
return f
return None
def get_execution_data(
self, context: "ScheduleExecutionContext"
) -> List[Union[RunRequest, SkipReason]]:
check.inst_param(context, "context", ScheduleExecutionContext)
execution_fn = cast(Callable[[ScheduleExecutionContext], Any],
self._execution_fn)
result = list(ensure_gen(execution_fn(context)))
if not result:
return []
if len(result) == 1:
check.is_list(result, of_type=(RunRequest, SkipReason))
data = result[0]
if isinstance(data, SkipReason):
return result
check.inst(data, RunRequest)
return [
RunRequest(
run_key=data.run_key,
run_config=data.run_config,
tags=merge_dicts(data.tags,
PipelineRun.tags_for_schedule(self)),
)
]
check.is_list(result, of_type=RunRequest)
check.invariant(
not any(not data.run_key for data in result),
"Schedules that return multiple RunRequests must specify a run_key in each RunRequest",
)
# clone all the run requests with the required schedule tags
return [
RunRequest(
run_key=data.run_key,
run_config=data.run_config,
tags=merge_dicts(data.tags,
PipelineRun.tags_for_schedule(self)),
) for data in result
]
def get_execution_data(self, context):
check.inst_param(context, "context", SensorExecutionContext)
result = list(ensure_gen(self._evaluation_fn(context)))
if not result or result == [None]:
run_requests = []
skip_message = None
elif len(result) == 1:
item = result[0]
check.inst(item, (SkipReason, RunRequest))
run_requests = [item] if isinstance(item, RunRequest) else []
skip_message = item.skip_message if isinstance(
item, SkipReason) else None
else:
check.is_list(result, of_type=RunRequest)
run_requests = result
skip_message = None
return SensorExecutionData(run_requests, skip_message, context.cursor)
def from_dict(cls, d: Dict[str, Any]) -> "NodeResult":
"""Constructs an instance of :class:`NodeResult <dagster_dbt.NodeResult>` from a dictionary.
Args:
d (Dict[str, Any]): A dictionary with key-values to construct a :class:`NodeResult
<dagster_dbt.NodeResult>`.
Returns:
NodeResult: An instance of :class:`NodeResult <dagster_dbt.NodeResult>`.
"""
check.dict_elem(d, "node")
check.opt_str_elem(d, "error")
check.float_elem(d, "execution_time")
check.opt_str_elem(d, "thread_id")
check.list_elem(d, "timing")
check.is_list(d["timing"], of_type=Dict)
check.opt_dict_elem(d, "table")
return cls(step_timings=d.get("timing"), **d)
def __init__(
self,
name=None,
dagster_type=None,
description=None,
default_value=NoValueSentinel,
root_manager_key=None,
metadata=None,
asset_key=None,
asset_partitions=None,
# when adding new params, make sure to update combine_with_inferred below
):
self._name = check_valid_name(name) if name else None
self._type_not_set = dagster_type is None
self._dagster_type = check.inst(resolve_dagster_type(dagster_type), DagsterType)
self._description = check.opt_str_param(description, "description")
self._default_value = _check_default_value(self._name, self._dagster_type, default_value)
if root_manager_key:
experimental_arg_warning("root_manager_key", "InputDefinition.__init__")
self._root_manager_key = check.opt_str_param(root_manager_key, "root_manager_key")
self._metadata = check.opt_dict_param(metadata, "metadata", key_type=str)
self._metadata_entries = check.is_list(
normalize_metadata(self._metadata, [], allow_invalid=True), MetadataEntry
)
if asset_key:
experimental_arg_warning("asset_key", "InputDefinition.__init__")
if not callable(asset_key):
check.opt_inst_param(asset_key, "asset_key", AssetKey)
self._asset_key = asset_key
if asset_partitions:
experimental_arg_warning("asset_partitions", "InputDefinition.__init__")
check.param_invariant(
asset_key is not None,
"asset_partitions",
'Cannot specify "asset_partitions" argument without also specifying "asset_key"',
)
if callable(asset_partitions):
self._asset_partitions_fn = asset_partitions
elif asset_partitions is not None:
asset_partitions = check.opt_set_param(asset_partitions, "asset_partitions", str)
self._asset_partitions_fn = lambda _: asset_partitions
else:
self._asset_partitions_fn = None
def __new__(
cls,
asset_key: Union[List[str], AssetKey, str],
description: Optional[str] = None,
metadata_entries: Optional[List[EventMetadataEntry]] = None,
partition: Optional[str] = None,
tags: Optional[Dict[str, str]] = None,
metadata: Optional[Dict[str, MetadataValues]] = None,
):
if isinstance(asset_key, AssetKey):
check.inst_param(asset_key, "asset_key", AssetKey)
elif isinstance(asset_key, str):
asset_key = AssetKey(parse_asset_key_string(asset_key))
elif isinstance(asset_key, list):
check.is_list(asset_key, of_type=str)
asset_key = AssetKey(asset_key)
else:
check.is_tuple(asset_key, of_type=str)
asset_key = AssetKey(asset_key)
if tags:
experimental_class_param_warning("tags", "AssetMaterialization")
metadata = check.opt_dict_param(metadata, "metadata", key_type=str)
metadata_entries = check.opt_list_param(metadata_entries,
"metadata_entries",
of_type=EventMetadataEntry)
return super(AssetMaterialization, cls).__new__(
cls,
asset_key=asset_key,
description=check.opt_str_param(description, "description"),
metadata_entries=cast(List[EventMetadataEntry],
parse_metadata(metadata, metadata_entries)),
partition=check.opt_str_param(partition, "partition"),
tags=check.opt_dict_param(tags,
"tags",
key_type=str,
value_type=str),
)
def evaluate_tick(self, context: "SensorEvaluationContext") -> "SensorExecutionData":
"""Evaluate sensor using the provided context.
Args:
context (SensorEvaluationContext): The context with which to evaluate this sensor.
Returns:
SensorExecutionData: Contains list of run requests, or skip message if present.
"""
check.inst_param(context, "context", SensorEvaluationContext)
result = list(ensure_gen(self._evaluation_fn(context)))
if not result or result == [None]:
run_requests = []
pipeline_run_reactions = []
skip_message = None
elif len(result) == 1:
item = result[0]
check.inst(item, (SkipReason, RunRequest, PipelineRunReaction))
run_requests = [item] if isinstance(item, RunRequest) else []
pipeline_run_reactions = [item] if isinstance(item, PipelineRunReaction) else []
skip_message = item.skip_message if isinstance(item, SkipReason) else None
elif isinstance(result[0], RunRequest):
check.is_list(result, of_type=RunRequest)
run_requests = result
pipeline_run_reactions = []
skip_message = None
else:
run_requests = []
check.is_list(result, of_type=PipelineRunReaction)
pipeline_run_reactions = result
skip_message = None
return SensorExecutionData(
run_requests,
skip_message,
context.cursor,
pipeline_run_reactions,
)
def test_is_list():
assert check.is_list([]) == []
with pytest.raises(CheckError):
check.is_list(None)
with pytest.raises(CheckError):
check.is_list('3u4')
with pytest.raises(CheckError, match='Did you pass a class'):
check.is_list([str], of_type=int)
def test_typed_is_list():
class Foo:
pass
class Bar:
pass
assert check.is_list([], Foo) == []
foo_list = [Foo()]
assert check.is_list(foo_list, of_type=Foo) == foo_list
assert check.is_list([Foo(), Bar()], of_type=(Foo, Bar))
with pytest.raises(CheckError):
check.is_list([Bar()], of_type=Foo)
with pytest.raises(CheckError):
check.is_list([None], of_type=Foo)
with pytest.raises(CheckError):
check.is_list([Foo(), Bar(), ""], of_type=(Foo, Bar))
def inner_type_key(self) -> str:
"""For container types such as Array or Noneable, the contained type. For a Map, the value type."""
# valid for Noneable, Map, and Array
check.invariant(
self.kind == ConfigTypeKind.NONEABLE
or self.kind == ConfigTypeKind.ARRAY
or self.kind == ConfigTypeKind.MAP
)
type_param_keys = check.is_list(self.type_param_keys, of_type=str)
if self.kind == ConfigTypeKind.MAP:
# For a Map, the inner (value) type is the second entry (the first is the key type)
check.invariant(len(type_param_keys) == 2)
return type_param_keys[1]
else:
check.invariant(len(type_param_keys) == 1)
return type_param_keys[0]
def from_dict(cls, d: Dict[str, Any]) -> "NodeResult":
"""Constructs an instance of :class:`NodeResult <dagster_dbt.NodeResult>` from a dictionary.
Args:
d (Dict[str, Any]): A dictionary with key-values to construct a :class:`NodeResult
<dagster_dbt.NodeResult>`.
Returns:
NodeResult: An instance of :class:`NodeResult <dagster_dbt.NodeResult>`.
"""
# When executing from the CLI in 0.19.x, we get unique_id as a top level attribute
if "unique_id" in d:
unique_id = check.str_elem(d, "unique_id")
node = None
# When executing via RPC server or via CLI in 0.18.x or lower, we get unique id within
# "node" schema
else:
node = check.dict_elem(d, "node")
unique_id = check.str_elem(node, "unique_id")
error = check.opt_str_elem(d, "error")
execution_time = check.float_elem(d, "execution_time")
thread_id = check.opt_str_elem(d, "thread_id")
check.list_elem(d, "timing")
step_timings = [
StepTiming(
name=d["name"],
started_at=parser.isoparse(d["started_at"]),
completed_at=parser.isoparse(d["completed_at"]),
) for d in check.is_list(d["timing"], of_type=Dict)
]
table = check.opt_dict_elem(d, "table")
return cls(
node=node,
unique_id=unique_id,
step_timings=step_timings,
error=error,
execution_time=execution_time,
thread_id=thread_id,
table=table,
)
def test_typed_is_list():
class Foo(object):
pass
class Bar(object):
pass
assert check.is_list([], Foo) == []
foo_list = [Foo()]
assert check.is_list(foo_list, Foo) == foo_list
with pytest.raises(CheckError):
check.is_list([Bar()], Foo)
with pytest.raises(CheckError):
check.is_list([None], Foo)
def non_scalar_type_key(self) -> str:
check.invariant(self.kind == ConfigTypeKind.SCALAR_UNION)
type_param_keys = check.is_list(self.type_param_keys, of_type=str)
return type_param_keys[1]
def field_names(self) -> List[str]:
fields = check.is_list(self.fields, of_type=ConfigFieldSnap)
return [fs.name for fs in fields]
def __init__(
self,
dagster_type=None,
name: Optional[str] = None,
description: Optional[str] = None,
is_required: bool = True,
io_manager_key: Optional[str] = None,
metadata: Optional[MetadataUserInput] = None,
asset_key: Optional[Union[AssetKey, DynamicAssetKey]] = None,
asset_partitions: Optional[Union[AbstractSet[str],
Callable[["OutputContext"],
AbstractSet[str]]]] = None,
asset_partitions_def: Optional["PartitionsDefinition"] = None
# make sure new parameters are updated in combine_with_inferred below
):
from dagster.core.definitions.partition import PartitionsDefinition
self._name = check_valid_name(
check.opt_str_param(name, "name", DEFAULT_OUTPUT))
self._type_not_set = dagster_type is None
self._dagster_type = resolve_dagster_type(dagster_type)
self._description = check.opt_str_param(description, "description")
self._is_required = check.bool_param(is_required, "is_required")
self._io_manager_key = check.opt_str_param(
io_manager_key,
"io_manager_key",
default="io_manager",
)
self._metadata = check.opt_dict_param(metadata,
"metadata",
key_type=str)
self._metadata_entries = check.is_list(
normalize_metadata(self._metadata, [], allow_invalid=True),
MetadataEntry)
if asset_key:
experimental_arg_warning("asset_key", "OutputDefinition.__init__")
if callable(asset_key):
warnings.warn(
"Passing a function as the `asset_key` argument to `Out` or `OutputDefinition` is "
"deprecated behavior and will be removed in version 0.15.0.")
else:
check.opt_inst_param(asset_key, "asset_key", AssetKey)
self._asset_key = asset_key
if asset_partitions:
experimental_arg_warning("asset_partitions",
"OutputDefinition.__init__")
check.param_invariant(
asset_key is not None,
"asset_partitions",
'Cannot specify "asset_partitions" argument without also specifying "asset_key"',
)
self._asset_partitions_fn: Optional[Callable[["OutputContext"],
AbstractSet[str]]]
if callable(asset_partitions):
self._asset_partitions_fn = asset_partitions
elif asset_partitions is not None:
asset_partitions = check.opt_set_param(asset_partitions,
"asset_partitions", str)
def _fn(_context: "OutputContext") -> AbstractSet[str]:
return cast(AbstractSet[str], asset_partitions) # mypy bug?
self._asset_partitions_fn = _fn
else:
self._asset_partitions_fn = None
if asset_partitions_def:
experimental_arg_warning("asset_partitions_def",
"OutputDefinition.__init__")
self._asset_partitions_def = check.opt_inst_param(
asset_partitions_def, "asset_partition_def", PartitionsDefinition)
def _execution_fn(context):
check.inst_param(context, "context", ScheduleEvaluationContext)
with user_code_error_boundary(
ScheduleExecutionError,
lambda:
f"Error occurred during the execution of partition_selector for schedule {schedule_name}",
):
selector_result = partition_selector(context, self)
if isinstance(selector_result, SkipReason):
yield selector_result
return
selected_partitions = (selector_result if isinstance(
selector_result, (frozenlist, list)) else [selector_result])
check.is_list(selected_partitions, of_type=Partition)
if not selected_partitions:
yield SkipReason(
"Partition selector returned an empty list of partitions.")
return
missing_partition_names = [
partition.name for partition in selected_partitions
if partition.name not in self.get_partition_names(
context.scheduled_execution_time)
]
if missing_partition_names:
yield SkipReason(
"Partition selector returned partition" +
("s" if len(missing_partition_names) > 1 else "") +
f" not in the partition set: {', '.join(missing_partition_names)}."
)
return
with user_code_error_boundary(
ScheduleExecutionError,
lambda:
f"Error occurred during the execution of should_execute for schedule {schedule_name}",
):
if should_execute and not should_execute(context):
yield SkipReason(
"should_execute function for {schedule_name} returned false."
.format(schedule_name=schedule_name))
return
for selected_partition in selected_partitions:
with user_code_error_boundary(
ScheduleExecutionError,
lambda:
f"Error occurred during the execution of run_config_fn for schedule {schedule_name}",
):
run_config = self.run_config_for_partition(
selected_partition)
with user_code_error_boundary(
ScheduleExecutionError,
lambda:
f"Error occurred during the execution of tags_fn for schedule {schedule_name}",
):
tags = self.tags_for_partition(selected_partition)
yield RunRequest(
run_key=selected_partition.name
if len(selected_partitions) > 0 else None,
run_config=run_config,
tags=tags,
)
def evaluate_tick(
self, context: "SensorEvaluationContext") -> "SensorExecutionData":
"""Evaluate sensor using the provided context.
Args:
context (SensorEvaluationContext): The context with which to evaluate this sensor.
Returns:
SensorExecutionData: Contains list of run requests, or skip message if present.
"""
check.inst_param(context, "context", SensorEvaluationContext)
result = list(ensure_gen(self._evaluation_fn(context)))
skip_message: Optional[str] = None
run_requests: List[RunRequest]
pipeline_run_reactions: List[PipelineRunReaction]
if not result or result == [None]:
run_requests = []
pipeline_run_reactions = []
skip_message = "Sensor function returned an empty result"
elif len(result) == 1:
item = result[0]
check.inst(item, (SkipReason, RunRequest, PipelineRunReaction))
run_requests = [item] if isinstance(item, RunRequest) else []
pipeline_run_reactions = ([cast(
PipelineRunReaction, item)] if isinstance(
item, PipelineRunReaction) else [])
skip_message = item.skip_message if isinstance(
item, SkipReason) else None
else:
check.is_list(result,
(SkipReason, RunRequest, PipelineRunReaction))
has_skip = any(map(lambda x: isinstance(x, SkipReason), result))
has_run_request = any(
map(lambda x: isinstance(x, RunRequest), result))
has_run_reaction = any(
map(lambda x: isinstance(x, PipelineRunReaction), result))
if has_skip:
if has_run_request:
check.failed(
"Expected a single SkipReason or one or more RunRequests: received both "
"RunRequest and SkipReason")
elif has_run_reaction:
check.failed(
"Expected a single SkipReason or one or more PipelineRunReaction: "
"received both PipelineRunReaction and SkipReason")
else:
check.failed(
"Expected a single SkipReason: received multiple SkipReasons"
)
if has_run_request:
run_requests = cast(List[RunRequest], result)
pipeline_run_reactions = []
else:
# only run reactions
run_requests = []
pipeline_run_reactions = cast(List[PipelineRunReaction],
result)
self.check_valid_run_requests(run_requests)
return SensorExecutionData(
run_requests,
skip_message,
context.cursor,
pipeline_run_reactions,
)
def __init__(
self,
dagster_type=None,
name=None,
description=None,
is_required=None,
io_manager_key=None,
metadata=None,
asset_key=None,
asset_partitions=None,
asset_partitions_def=None
# make sure new parameters are updated in combine_with_inferred below
):
from dagster.core.definitions.partition import PartitionsDefinition
self._name = (check_valid_name(
check.opt_str_param(name, "name", DEFAULT_OUTPUT))
if name is not NoNameSentinel else None)
self._type_not_set = dagster_type is None
self._dagster_type = resolve_dagster_type(dagster_type)
self._description = check.opt_str_param(description, "description")
self._is_required = check.opt_bool_param(is_required,
"is_required",
default=True)
self._manager_key = check.opt_str_param(io_manager_key,
"io_manager_key",
default="io_manager")
self._metadata = check.opt_dict_param(metadata,
"metadata",
key_type=str)
self._metadata_entries = check.is_list(
normalize_metadata(self._metadata, [], allow_invalid=True),
MetadataEntry)
if asset_key:
experimental_arg_warning("asset_key", "OutputDefinition.__init__")
if callable(asset_key):
warnings.warn(
"Passing a function as the `asset_key` argument to `Out` or `OutputDefinition` is "
"deprecated behavior and will be removed in version 0.15.0.")
else:
check.opt_inst_param(asset_key, "asset_key", AssetKey)
self._asset_key = asset_key
if asset_partitions:
experimental_arg_warning("asset_partitions",
"OutputDefinition.__init__")
check.param_invariant(
asset_key is not None,
"asset_partitions",
'Cannot specify "asset_partitions" argument without also specifying "asset_key"',
)
if callable(asset_partitions):
self._asset_partitions_fn = asset_partitions
elif asset_partitions is not None:
asset_partitions = check.opt_set_param(asset_partitions,
"asset_partitions", str)
self._asset_partitions_fn = lambda _: asset_partitions
else:
self._asset_partitions_fn = None
if asset_partitions_def:
experimental_arg_warning("asset_partitions_def",
"OutputDefinition.__init__")
self._asset_partitions_def = check.opt_inst_param(
asset_partitions_def, "asset_partition_def", PartitionsDefinition)
