def __init__(self, execution_context_data, log_manager, step):
from dagster.core.execution.plan.objects import ExecutionStep
from dagster.core.execution.resources_init import get_required_resource_keys_for_step
self._step = check.inst_param(step, 'step', ExecutionStep)
super(SystemStepExecutionContext,
self).__init__(execution_context_data, log_manager)
self._required_resource_keys = get_required_resource_keys_for_step(
step,
execution_context_data.pipeline_def,
execution_context_data.system_storage_def,
)
self._resources = self._execution_context_data.scoped_resources_builder.build(
self._required_resource_keys)
step_launcher_resources = [
resource for resource in self._resources
if isinstance(resource, StepLauncher)
]
if len(step_launcher_resources) > 1:
raise DagsterInvariantViolationError(
'Multiple required resources for solid {solid_name} have inherit StepLauncher'
'There should be at most one step launcher resource per solid.'
.format(solid_name=step.solid_handle.name))
elif len(step_launcher_resources) == 1:
self._step_launcher = step_launcher_resources[0]
else:
self._step_launcher = None
self._log_manager = log_manager
def __init__(
self,
execution_context_data: SystemExecutionContextData,
log_manager: DagsterLogManager,
step: ExecutionStep,
):
from dagster.core.execution.resources_init import get_required_resource_keys_for_step
self._step = check.inst_param(step, "step", ExecutionStep)
super(SystemStepExecutionContext,
self).__init__(execution_context_data, log_manager)
self._required_resource_keys = get_required_resource_keys_for_step(
step,
execution_context_data.execution_plan,
execution_context_data.intermediate_storage_def,
)
self._resources = self._execution_context_data.scoped_resources_builder.build(
self._required_resource_keys)
step_launcher_resources = [
resource for resource in self._resources
if isinstance(resource, StepLauncher)
]
self._step_launcher: Optional[StepLauncher] = None
if len(step_launcher_resources) > 1:
raise DagsterInvariantViolationError(
"Multiple required resources for solid {solid_name} have inherit StepLauncher"
"There should be at most one step launcher resource per solid."
.format(solid_name=step.solid_handle.name))
elif len(step_launcher_resources) == 1:
self._step_launcher = step_launcher_resources[0]
self._log_manager = log_manager
def __init__(
self,
plan_data: PlanData,
execution_data: ExecutionData,
log_manager: DagsterLogManager,
step: ExecutionStep,
output_capture: Optional[Dict[StepOutputHandle, Any]],
previous_attempt_count: int,
):
from dagster.core.execution.resources_init import get_required_resource_keys_for_step
super(StepExecutionContext, self).__init__(
plan_data=plan_data,
execution_data=execution_data,
log_manager=log_manager,
output_capture=output_capture,
)
self._step = step
self._required_resource_keys = get_required_resource_keys_for_step(
plan_data.pipeline.get_definition(),
step,
plan_data.execution_plan,
)
self._resources = execution_data.scoped_resources_builder.build(
self._required_resource_keys
)
self._previous_attempt_count = previous_attempt_count
self._input_lineage: List[AssetLineageInfo] = []
resources_iter = cast(Iterable, self._resources)
step_launcher_resources = [
resource for resource in resources_iter if isinstance(resource, StepLauncher)
]
self._step_launcher: Optional[StepLauncher] = None
if len(step_launcher_resources) > 1:
raise DagsterInvariantViolationError(
"Multiple required resources for {described_op} have inherited StepLauncher"
"There should be at most one step launcher resource per {node_type}.".format(
described_op=self.describe_op(), node_type=self.solid_def.node_type_str
)
)
elif len(step_launcher_resources) == 1:
self._step_launcher = step_launcher_resources[0]
self._step_exception: Optional[BaseException] = None
self._step_output_capture: Optional[Dict[StepOutputHandle, Any]] = None
# Enable step output capture if there are any hooks which will receive them.
# Expect in the future that hooks may control whether or not they get outputs,
# but for now presence of any will cause output capture.
if self.pipeline_def.get_all_hooks_for_handle(self.solid_handle):
self._step_output_capture = {}
self._output_metadata: Dict[str, Any] = {}
self._seen_outputs: Dict[str, Union[str, Set[str]]] = {}
def __init__(
self,
plan_data: PlanData,
execution_data: ExecutionData,
log_manager: DagsterLogManager,
step: ExecutionStep,
output_capture: Optional[Dict[StepOutputHandle, Any]],
previous_attempt_count: int,
):
from dagster.core.execution.resources_init import get_required_resource_keys_for_step
super(StepExecutionContext, self).__init__(
plan_data=plan_data,
execution_data=execution_data,
log_manager=log_manager,
output_capture=output_capture,
)
self._step = step
self._required_resource_keys = get_required_resource_keys_for_step(
plan_data.pipeline.get_definition(),
step,
plan_data.execution_plan,
execution_data.environment_config,
execution_data.intermediate_storage_def,
)
self._resources = execution_data.scoped_resources_builder.build(
self._required_resource_keys
)
self._previous_attempt_count = previous_attempt_count
resources_iter = cast(Iterable, self._resources)
step_launcher_resources = [
resource for resource in resources_iter if isinstance(resource, StepLauncher)
]
self._step_launcher: Optional[StepLauncher] = None
if len(step_launcher_resources) > 1:
raise DagsterInvariantViolationError(
"Multiple required resources for solid {solid_name} have inherit StepLauncher"
"There should be at most one step launcher resource per solid.".format(
solid_name=step.solid_handle.name
)
)
elif len(step_launcher_resources) == 1:
self._step_launcher = step_launcher_resources[0]
self._step_exception = None
self._step_output_capture: Dict[StepOutputHandle, Any] = {}
def __init__(self, pipeline_context_data, log_manager, step):
from dagster.core.execution.plan.objects import ExecutionStep
from dagster.core.execution.resources_init import get_required_resource_keys_for_step
self._step = check.inst_param(step, 'step', ExecutionStep)
super(SystemStepExecutionContext,
self).__init__(pipeline_context_data, log_manager)
self._required_resource_keys = get_required_resource_keys_for_step(
step,
pipeline_context_data.pipeline_def,
pipeline_context_data.system_storage_def,
)
self._resources = self._pipeline_context_data.scoped_resources_builder.build(
self._required_resource_keys)
self._log_manager = log_manager
