def single_resource_event_generator(context, resource_name, resource_def):
try:
msg_fn = lambda: "Error executing resource_fn on ResourceDefinition {name}".format(
name=resource_name)
with user_code_error_boundary(DagsterResourceFunctionError, msg_fn):
try:
with time_execution_scope() as timer_result:
resource_or_gen = resource_def.resource_fn(context)
gen = ensure_gen(resource_or_gen)
resource = next(gen)
resource = InitializedResource(
resource, format_duration(timer_result.millis))
except StopIteration:
check.failed(
"Resource generator {name} must yield one item.".format(
name=resource_name))
yield resource
except DagsterUserCodeExecutionError as dagster_user_error:
raise dagster_user_error
with user_code_error_boundary(DagsterResourceFunctionError, msg_fn):
try:
next(gen)
except StopIteration:
pass
else:
check.failed(
"Resource generator {name} yielded more than one item.".format(
name=resource_name))
def _set_addressable_asset(context, step_output_handle, asset_store_handle,
value):
check.inst_param(asset_store_handle, "asset_store_handle",
AssetStoreHandle)
asset_store = context.get_asset_store(asset_store_handle.asset_store_key)
materializations = asset_store.set_asset(context, step_output_handle,
value,
asset_store_handle.asset_metadata)
# Allow zero, one, or multiple AssetMaterialization yielded by set_asset
if materializations is not None:
for materialization in ensure_gen(materializations):
if not isinstance(materialization, AssetMaterialization):
raise DagsterInvariantViolationError((
"asset_store on output {output_name} has returned "
"value {value} of type {python_type}. The return type can only be "
"AssetMaterialization.").format(
output_name=step_output_handle.output_name,
value=repr(materialization),
python_type=type(materialization).__name__,
))
yield materialization
# SET_ASSET operation by AssetStore
yield AssetStoreOperation(AssetStoreOperationType.SET_ASSET,
step_output_handle, asset_store_handle)
def load_input_object(
self,
step_context: "StepExecutionContext",
input_def: InputDefinition,
):
from dagster.core.events import DagsterEvent
values = []
# some upstream steps may have skipped and we allow fan-in to continue in their absence
source_handles_to_skip = list(
filter(lambda x: not step_context.can_load(x),
self.step_output_handle_dependencies))
for inner_source in self.sources:
if (isinstance(inner_source, FromStepOutput) and
inner_source.step_output_handle in source_handles_to_skip):
continue
for event_or_input_value in ensure_gen(
inner_source.load_input_object(step_context, input_def)):
if isinstance(event_or_input_value, DagsterEvent):
yield event_or_input_value
else:
values.append(event_or_input_value)
yield values
def _steps_execution_iterator(pipeline_context, execution_plan, run_config, step_keys_to_execute):
'''Iterates over execution of individual steps yielding the associated events.
Does not yield pipeline level events asside from init failure when the context fails to construct.
'''
check.inst_param(
pipeline_context, 'pipeline_context', (DagsterEvent, SystemPipelineExecutionContext)
)
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
check.inst_param(run_config, 'run_config', RunConfig)
check.list_param(step_keys_to_execute, 'step_keys_to_execute', of_type=str)
if (
isinstance(pipeline_context, DagsterEvent)
# pylint: disable=no-member
and pipeline_context.event_type == DagsterEventType.PIPELINE_INIT_FAILURE
):
return ensure_gen(pipeline_context)
_setup_reexecution(run_config, pipeline_context, execution_plan)
# Engine execution returns a generator of yielded events, so returning here means this function
# also returns a generator
return pipeline_context.executor_config.get_engine().execute(
pipeline_context, execution_plan, step_keys_to_execute
)
def _wrapped_fn(context: "ScheduleEvaluationContext"):
if should_execute:
with user_code_error_boundary(
ScheduleExecutionError,
lambda: f"Error occurred during the execution of should_execute for schedule {schedule_name}",
):
if not should_execute(context):
yield SkipReason(
f"should_execute function for {schedule_name} returned false."
)
return
with user_code_error_boundary(
ScheduleExecutionError,
lambda: f"Error occurred during the evaluation of schedule {schedule_name}",
):
result = fn(context) if has_context_arg else fn()
if isinstance(result, dict):
# this is the run-config based decorated function, wrap the evaluated run config
# and tags in a RunRequest
evaluated_run_config = copy.deepcopy(result)
evaluated_tags = _tags_fn(context) if _tags_fn else None
yield RunRequest(
run_key=None,
run_config=evaluated_run_config,
tags=evaluated_tags,
)
else:
# this is a run-request based decorated function
yield from ensure_gen(result)
def user_code_context_manager(user_fn, error_cls, msg_fn):
'''Wraps the output of a user provided function that may yield or return a value and
returns a generator that asserts it only yields a single value.
'''
check.callable_param(user_fn, 'user_fn')
check.subclass_param(error_cls, 'error_cls', DagsterUserCodeExecutionError)
with user_code_error_boundary(error_cls, msg_fn):
thing_or_gen = user_fn()
gen = ensure_gen(thing_or_gen)
try:
thing = next(gen)
except StopIteration:
check.failed('Must yield one item. You did not yield anything.')
yield thing
stopped = False
try:
next(gen)
except StopIteration:
stopped = True
check.invariant(stopped,
'Must yield one item. Yielded more than one item')
def load_input_parameter(self, input_name: str):
# load input from source
step_context = self.context._step_context # pylint: disable=protected-access
step_input = step_context.step.step_input_named(input_name)
for event_or_input_value in ensure_gen(
step_input.source.load_input_object(step_context)):
if isinstance(event_or_input_value, DagsterEvent):
continue
else:
return event_or_input_value
def get_execution_data(self, context):
check.inst_param(context, "context", SensorExecutionContext)
result = list(ensure_gen(self._evaluation_fn(context)))
if not result or result == [None]:
return []
if len(result) == 1:
return check.is_list(result, of_type=(RunRequest, SkipReason))
return check.is_list(result, of_type=RunRequest)
def _steps_execution_iterator(pipeline_context, execution_plan, run_config,
step_keys_to_execute):
'''Iterates over execution of individual steps yielding the associated events.
Does not yield pipeline level events asside from init failure when the context fails to construct.
'''
check.inst_param(pipeline_context, 'pipeline_context',
(DagsterEvent, SystemPipelineExecutionContext))
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
check.inst_param(run_config, 'run_config', RunConfig)
check.opt_list_param(step_keys_to_execute,
'step_keys_to_execute',
of_type=str)
if (isinstance(pipeline_context, DagsterEvent)
and pipeline_context.event_type # pylint: disable=no-member
== DagsterEventType.PIPELINE_INIT_FAILURE):
return ensure_gen(pipeline_context)
if not step_keys_to_execute:
step_keys_to_execute = [
step.key for step in execution_plan.topological_steps()
]
if not step_keys_to_execute:
pipeline_context.log.debug(
'Pipeline {pipeline} has no steps to execute and no execution will happen'
.format(pipeline=pipeline_context.pipeline_def.display_name))
return ensure_gen(DagsterEvent.pipeline_success(pipeline_context))
else:
for step_key in step_keys_to_execute:
if not execution_plan.has_step(step_key):
raise DagsterExecutionStepNotFoundError(
'Execution plan does not contain step \'{}\''.format(
step_key),
step_key=step_key,
)
_setup_reexecution(run_config, pipeline_context, execution_plan)
return _invoke_executor_on_plan(pipeline_context, execution_plan,
step_keys_to_execute)
def _execute_plan_iterator(pipeline_context, execution_plan, run_config,
step_keys_to_execute):
check.inst_param(pipeline_context, 'pipeline_context',
(DagsterEvent, SystemPipelineExecutionContext))
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
check.inst_param(run_config, 'run_config', RunConfig)
check.opt_list_param(step_keys_to_execute,
'step_keys_to_execute',
of_type=str)
if (isinstance(pipeline_context, DagsterEvent)
and pipeline_context.event_type # pylint: disable=no-member
== DagsterEventType.PIPELINE_INIT_FAILURE):
return ensure_gen(pipeline_context)
if not step_keys_to_execute:
step_keys_to_execute = [
step.key for step in execution_plan.topological_steps()
]
if not step_keys_to_execute:
pipeline_context.log.debug(
'Pipeline {pipeline} has no steps to execute and no execution will happen'
.format(pipeline=pipeline_context.pipeline_def.display_name))
return ensure_gen(DagsterEvent.pipeline_success(pipeline_context))
else:
for step_key in step_keys_to_execute:
if not execution_plan.has_step(step_key):
raise DagsterExecutionStepNotFoundError(
'Execution plan does not contain step "{}"'.format(
step_key),
step_key=step_key)
_setup_reexecution(run_config, pipeline_context, execution_plan)
return _invoke_executor_on_plan(pipeline_context, execution_plan,
step_keys_to_execute)
def _materializations_to_events(step_context, step_output_handle,
materializations):
if materializations is not None:
for materialization in ensure_gen(materializations):
if not isinstance(materialization, AssetMaterialization):
raise DagsterInvariantViolationError((
"IO manager on output {output_name} has returned "
"value {value} of type {python_type}. The return type can only be "
"AssetMaterialization.").format(
output_name=step_output_handle.output_name,
value=repr(materialization),
python_type=type(materialization).__name__,
))
yield DagsterEvent.step_materialization(step_context,
materialization)
def _steps_execution_iterator(pipeline_context, execution_plan, pipeline_run):
'''Iterates over execution of individual steps yielding the associated events.
'''
check.inst_param(pipeline_context, 'pipeline_context',
(DagsterEvent, SystemPipelineExecutionContext))
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
if (isinstance(pipeline_context, DagsterEvent)
# pylint: disable=no-member
and pipeline_context.event_type
== DagsterEventType.PIPELINE_INIT_FAILURE):
return ensure_gen(pipeline_context)
if execution_plan.previous_run_id:
validate_retry_memoization(pipeline_context, execution_plan)
return pipeline_context.executor_config.get_engine().execute(
pipeline_context, execution_plan)
def get_execution_data(self, context):
check.inst_param(context, "context", SensorExecutionContext)
result = list(ensure_gen(self._evaluation_fn(context)))
if not result or result == [None]:
run_requests = []
skip_message = None
elif len(result) == 1:
item = result[0]
check.inst(item, (SkipReason, RunRequest))
run_requests = [item] if isinstance(item, RunRequest) else []
skip_message = item.skip_message if isinstance(
item, SkipReason) else None
else:
check.is_list(result, of_type=RunRequest)
run_requests = result
skip_message = None
return SensorExecutionData(run_requests, skip_message, context.cursor)
def get_execution_data(
self, context: "ScheduleExecutionContext"
) -> List[Union[RunRequest, SkipReason]]:
check.inst_param(context, "context", ScheduleExecutionContext)
execution_fn = cast(Callable[[ScheduleExecutionContext], Any],
self._execution_fn)
result = list(ensure_gen(execution_fn(context)))
if not result:
return []
if len(result) == 1:
check.is_list(result, of_type=(RunRequest, SkipReason))
data = result[0]
if isinstance(data, SkipReason):
return result
check.inst(data, RunRequest)
return [
RunRequest(
run_key=data.run_key,
run_config=data.run_config,
tags=merge_dicts(data.tags,
PipelineRun.tags_for_schedule(self)),
)
]
check.is_list(result, of_type=RunRequest)
check.invariant(
not any(not data.run_key for data in result),
"Schedules that return multiple RunRequests must specify a run_key in each RunRequest",
)
# clone all the run requests with the required schedule tags
return [
RunRequest(
run_key=data.run_key,
run_config=data.run_config,
tags=merge_dicts(data.tags,
PipelineRun.tags_for_schedule(self)),
) for data in result
]
def _wrapped_resource_iterator(resource_or_gen):
"""Returns an iterator which yields a single item, which is the resource.
If the resource is not a context manager, then resource teardown happens following the first yield.
If the resource is a context manager, then resource initialization happens as the passed-in
context manager opens. Resource teardown happens as the passed-in context manager closes (which will occur after all compute is finished).
"""
# Context managers created using contextlib.contextdecorator are not usable as iterators.
# Opening context manager and directly yielding preserves initialization/teardown behavior,
# while also letting the context manager be used as an iterator.
if isinstance(resource_or_gen, ContextDecorator):
def _gen_resource():
with resource_or_gen as resource:
yield resource
return _gen_resource()
# Otherwise, coerce to generator without opening context manager
return ensure_gen(resource_or_gen)
def evaluate_tick(self, context: "SensorEvaluationContext") -> "SensorExecutionData":
"""Evaluate sensor using the provided context.
Args:
context (SensorEvaluationContext): The context with which to evaluate this sensor.
Returns:
SensorExecutionData: Contains list of run requests, or skip message if present.
"""
check.inst_param(context, "context", SensorEvaluationContext)
result = list(ensure_gen(self._evaluation_fn(context)))
if not result or result == [None]:
run_requests = []
pipeline_run_reactions = []
skip_message = None
elif len(result) == 1:
item = result[0]
check.inst(item, (SkipReason, RunRequest, PipelineRunReaction))
run_requests = [item] if isinstance(item, RunRequest) else []
pipeline_run_reactions = [item] if isinstance(item, PipelineRunReaction) else []
skip_message = item.skip_message if isinstance(item, SkipReason) else None
elif isinstance(result[0], RunRequest):
check.is_list(result, of_type=RunRequest)
run_requests = result
pipeline_run_reactions = []
skip_message = None
else:
run_requests = []
check.is_list(result, of_type=PipelineRunReaction)
pipeline_run_reactions = result
skip_message = None
return SensorExecutionData(
run_requests,
skip_message,
context.cursor,
pipeline_run_reactions,
)
def load_input_object(self, step_context):
from dagster.core.events import DagsterEvent
values = []
# some upstream steps may have skipped and we allow fan-in to continue in their absence
source_handles_to_skip = self._step_output_handles_no_output(
step_context)
for inner_source in self.sources:
if (inner_source.step_output_handle_dependencies and
inner_source.step_output_handle in source_handles_to_skip):
continue
for event_or_input_value in ensure_gen(
inner_source.load_input_object(step_context)):
if isinstance(event_or_input_value, DagsterEvent):
yield event_or_input_value
else:
values.append(event_or_input_value)
yield values
def single_resource_event_generator(context, resource_name, resource_def):
try:
msg_fn = lambda: "Error executing resource_fn on ResourceDefinition {name}".format(
name=resource_name)
with user_code_error_boundary(DagsterResourceFunctionError, msg_fn):
try:
with time_execution_scope() as timer_result:
resource_or_gen = (resource_def.resource_fn(context)
if is_context_provided(
get_function_params(
resource_def.resource_fn)) else
resource_def.resource_fn())
# Flag for whether resource is generator. This is used to ensure that teardown
# occurs when resources are initialized out of execution.
is_gen = inspect.isgenerator(resource_or_gen)
gen = ensure_gen(resource_or_gen)
resource = next(gen)
resource = InitializedResource(
resource, format_duration(timer_result.millis), is_gen)
except StopIteration:
check.failed(
"Resource generator {name} must yield one item.".format(
name=resource_name))
yield resource
except DagsterUserCodeExecutionError as dagster_user_error:
raise dagster_user_error
with user_code_error_boundary(DagsterResourceFunctionError, msg_fn):
try:
next(gen)
except StopIteration:
pass
else:
check.failed(
"Resource generator {name} yielded more than one item.".format(
name=resource_name))
def single_resource_event_generator(context, resource_name, resource_def):
try:
msg_fn = lambda: 'Error executing resource_fn on ResourceDefinition {name}'.format(
name=resource_name)
with user_code_error_boundary(DagsterResourceFunctionError, msg_fn):
try:
resource_or_gen = resource_def.resource_fn(context)
gen = ensure_gen(resource_or_gen)
resource = next(gen)
yield InitializedResource(resource)
except StopIteration:
check.failed(
'Resource generator {name} must yield one item.'.format(
name=resource_name))
try:
next(gen)
except StopIteration:
pass
else:
check.failed(
'Resource generator {name} yielded more than one item.'.
format(name=resource_name))
except DagsterUserCodeExecutionError as dagster_user_error:
raise dagster_user_error
def _store_output(
step_context: StepExecutionContext,
step_output_handle: StepOutputHandle,
output: Union[Output, DynamicOutput],
input_lineage: List[AssetLineageInfo],
) -> Iterator[DagsterEvent]:
output_def = step_context.solid_def.output_def_named(
step_output_handle.output_name)
output_manager = step_context.get_io_manager(step_output_handle)
output_context = step_context.get_output_context(step_output_handle)
with solid_execution_error_boundary(
DagsterExecutionHandleOutputError,
msg_fn=lambda:
(f'Error occurred while handling output "{output_context.name}" of '
f'step "{step_context.step.key}":'),
step_context=step_context,
step_key=step_context.step.key,
output_name=output_context.name,
):
handle_output_res = output_manager.handle_output(
output_context, output.value)
manager_materializations = []
manager_metadata_entries = []
if handle_output_res is not None:
for elt in ensure_gen(handle_output_res):
if isinstance(elt, AssetMaterialization):
manager_materializations.append(elt)
elif isinstance(elt, (EventMetadataEntry, PartitionMetadataEntry)):
experimental_functionality_warning(
"Yielding metadata from an IOManager's handle_output() function"
)
manager_metadata_entries.append(elt)
else:
raise DagsterInvariantViolationError(
f"IO manager on output {output_def.name} has returned "
f"value {elt} of type {type(elt).__name__}. The return type can only be "
"one of AssetMaterialization, EventMetadataEntry, PartitionMetadataEntry."
)
# do not alter explicitly created AssetMaterializations
for materialization in manager_materializations:
yield DagsterEvent.asset_materialization(step_context, materialization,
input_lineage)
asset_key, partitions = _asset_key_and_partitions_for_output(
output_context, output_def, output_manager)
if asset_key:
for materialization in _get_output_asset_materializations(
asset_key,
partitions,
output,
output_def,
manager_metadata_entries,
):
yield DagsterEvent.asset_materialization(step_context,
materialization,
input_lineage)
yield DagsterEvent.handled_output(
step_context,
output_name=step_output_handle.output_name,
manager_key=output_def.io_manager_key,
message_override=
f'Handled input "{step_output_handle.output_name}" using intermediate storage'
if isinstance(output_manager, IntermediateStorageAdapter) else None,
metadata_entries=[
entry for entry in manager_metadata_entries
if isinstance(entry, EventMetadataEntry)
],
)
def materialize_runtime_values(
self, context: "StepExecutionContext", config_value: object,
runtime_value: object
) -> Iterator[Union[Materialization, AssetMaterialization]]:
return ensure_gen(self._func(context, config_value, runtime_value))
def core_dagster_event_sequence_for_step(
step_context: StepExecutionContext,
) -> Iterator[DagsterEvent]:
"""
Execute the step within the step_context argument given the in-memory
events. This function yields a sequence of DagsterEvents, but without
catching any exceptions that have bubbled up during the computation
of the step.
"""
check.inst_param(step_context, "step_context", StepExecutionContext)
if step_context.previous_attempt_count > 0:
yield DagsterEvent.step_restarted_event(step_context, step_context.previous_attempt_count)
else:
yield DagsterEvent.step_start_event(step_context)
inputs = {}
for step_input in step_context.step.step_inputs:
input_def = step_input.source.get_input_def(step_context.pipeline_def)
dagster_type = input_def.dagster_type
if dagster_type.kind == DagsterTypeKind.NOTHING:
continue
for event_or_input_value in ensure_gen(step_input.source.load_input_object(step_context)):
if isinstance(event_or_input_value, DagsterEvent):
yield event_or_input_value
else:
check.invariant(step_input.name not in inputs)
inputs[step_input.name] = event_or_input_value
for input_name, input_value in inputs.items():
for evt in check.generator(
_type_checked_event_sequence_for_input(step_context, input_name, input_value)
):
yield evt
input_lineage = step_context.get_input_lineage()
# The core execution loop expects a compute generator in a specific format: a generator that
# takes a context and dictionary of inputs as input, yields output events. If a solid definition
# was generated from the @solid or @lambda_solid decorator, then compute_fn needs to be coerced
# into this format. If the solid definition was created directly, then it is expected that the
# compute_fn is already in this format.
if isinstance(step_context.solid_def.compute_fn, DecoratedSolidFunction):
core_gen = create_solid_compute_wrapper(step_context.solid_def)
else:
core_gen = step_context.solid_def.compute_fn
with time_execution_scope() as timer_result:
user_event_sequence = check.generator(
execute_core_compute(
step_context,
inputs,
core_gen,
)
)
# It is important for this loop to be indented within the
# timer block above in order for time to be recorded accurately.
for user_event in check.generator(
_step_output_error_checked_user_event_sequence(step_context, user_event_sequence)
):
if isinstance(user_event, DagsterEvent):
yield user_event
elif isinstance(user_event, (Output, DynamicOutput)):
for evt in _type_check_and_store_output(step_context, user_event, input_lineage):
yield evt
# for now, I'm ignoring AssetMaterializations yielded manually, but we might want
# to do something with these in the above path eventually
elif isinstance(user_event, (AssetMaterialization, Materialization)):
yield DagsterEvent.asset_materialization(step_context, user_event, input_lineage)
elif isinstance(user_event, AssetObservation):
yield DagsterEvent.asset_observation(step_context, user_event)
elif isinstance(user_event, ExpectationResult):
yield DagsterEvent.step_expectation_result(step_context, user_event)
else:
check.failed(
"Unexpected event {event}, should have been caught earlier".format(
event=user_event
)
)
yield DagsterEvent.step_success_event(
step_context, StepSuccessData(duration_ms=timer_result.millis)
)
def evaluate_tick(
self, context: "SensorEvaluationContext") -> "SensorExecutionData":
"""Evaluate sensor using the provided context.
Args:
context (SensorEvaluationContext): The context with which to evaluate this sensor.
Returns:
SensorExecutionData: Contains list of run requests, or skip message if present.
"""
check.inst_param(context, "context", SensorEvaluationContext)
result = list(ensure_gen(self._evaluation_fn(context)))
skip_message: Optional[str] = None
run_requests: List[RunRequest]
pipeline_run_reactions: List[PipelineRunReaction]
if not result or result == [None]:
run_requests = []
pipeline_run_reactions = []
skip_message = "Sensor function returned an empty result"
elif len(result) == 1:
item = result[0]
check.inst(item, (SkipReason, RunRequest, PipelineRunReaction))
run_requests = [item] if isinstance(item, RunRequest) else []
pipeline_run_reactions = ([cast(
PipelineRunReaction, item)] if isinstance(
item, PipelineRunReaction) else [])
skip_message = item.skip_message if isinstance(
item, SkipReason) else None
else:
check.is_list(result,
(SkipReason, RunRequest, PipelineRunReaction))
has_skip = any(map(lambda x: isinstance(x, SkipReason), result))
has_run_request = any(
map(lambda x: isinstance(x, RunRequest), result))
has_run_reaction = any(
map(lambda x: isinstance(x, PipelineRunReaction), result))
if has_skip:
if has_run_request:
check.failed(
"Expected a single SkipReason or one or more RunRequests: received both "
"RunRequest and SkipReason")
elif has_run_reaction:
check.failed(
"Expected a single SkipReason or one or more PipelineRunReaction: "
"received both PipelineRunReaction and SkipReason")
else:
check.failed(
"Expected a single SkipReason: received multiple SkipReasons"
)
if has_run_request:
run_requests = cast(List[RunRequest], result)
pipeline_run_reactions = []
else:
# only run reactions
run_requests = []
pipeline_run_reactions = cast(List[PipelineRunReaction],
result)
self.check_valid_run_requests(run_requests)
return SensorExecutionData(
run_requests,
skip_message,
context.cursor,
pipeline_run_reactions,
)
def test_ensure_gen():
zero = ensure_gen(0)
assert next(zero) == 0
with pytest.raises(StopIteration):
next(zero)
def materialize_runtime_values(self, context, config_value, runtime_value):
return ensure_gen(self._func(context, config_value, runtime_value))
def core_dagster_event_sequence_for_step(
step_context: SystemStepExecutionContext,
prior_attempt_count: int) -> Iterator[DagsterEvent]:
"""
Execute the step within the step_context argument given the in-memory
events. This function yields a sequence of DagsterEvents, but without
catching any exceptions that have bubbled up during the computation
of the step.
"""
check.inst_param(step_context, "step_context", SystemStepExecutionContext)
check.int_param(prior_attempt_count, "prior_attempt_count")
if prior_attempt_count > 0:
yield DagsterEvent.step_restarted_event(step_context,
prior_attempt_count)
else:
yield DagsterEvent.step_start_event(step_context)
inputs = {}
for step_input in step_context.step.step_inputs:
input_def = step_input.source.get_input_def(step_context.pipeline_def)
dagster_type = input_def.dagster_type
if dagster_type.kind == DagsterTypeKind.NOTHING:
continue
for event_or_input_value in ensure_gen(
step_input.source.load_input_object(step_context)):
if isinstance(event_or_input_value, DagsterEvent):
yield event_or_input_value
else:
check.invariant(step_input.name not in inputs)
inputs[step_input.name] = event_or_input_value
for input_name, input_value in inputs.items():
for evt in check.generator(
_type_checked_event_sequence_for_input(step_context,
input_name,
input_value)):
yield evt
with time_execution_scope() as timer_result:
user_event_sequence = check.generator(
_user_event_sequence_for_step_compute_fn(step_context, inputs))
# It is important for this loop to be indented within the
# timer block above in order for time to be recorded accurately.
for user_event in check.generator(
_step_output_error_checked_user_event_sequence(
step_context, user_event_sequence)):
if isinstance(user_event, (Output, DynamicOutput)):
for evt in _type_check_and_store_output(
step_context, user_event):
yield evt
elif isinstance(user_event,
(AssetMaterialization, Materialization)):
yield DagsterEvent.step_materialization(
step_context, user_event)
elif isinstance(user_event, ExpectationResult):
yield DagsterEvent.step_expectation_result(
step_context, user_event)
else:
check.failed(
"Unexpected event {event}, should have been caught earlier"
.format(event=user_event))
yield DagsterEvent.step_success_event(
step_context, StepSuccessData(duration_ms=timer_result.millis))
