def __init__(self, execution_context_data, log_manager, step): from dagster.core.execution.plan.objects import ExecutionStep from dagster.core.execution.resources_init import get_required_resource_keys_for_step self._step = check.inst_param(step, 'step', ExecutionStep) super(SystemStepExecutionContext, self).__init__(execution_context_data, log_manager) self._required_resource_keys = get_required_resource_keys_for_step( step, execution_context_data.pipeline_def, execution_context_data.system_storage_def, ) self._resources = self._execution_context_data.scoped_resources_builder.build( self._required_resource_keys) step_launcher_resources = [ resource for resource in self._resources if isinstance(resource, StepLauncher) ] if len(step_launcher_resources) > 1: raise DagsterInvariantViolationError( 'Multiple required resources for solid {solid_name} have inherit StepLauncher' 'There should be at most one step launcher resource per solid.' .format(solid_name=step.solid_handle.name)) elif len(step_launcher_resources) == 1: self._step_launcher = step_launcher_resources[0] else: self._step_launcher = None self._log_manager = log_manager
def __init__( self, execution_context_data: SystemExecutionContextData, log_manager: DagsterLogManager, step: ExecutionStep, ): from dagster.core.execution.resources_init import get_required_resource_keys_for_step self._step = check.inst_param(step, "step", ExecutionStep) super(SystemStepExecutionContext, self).__init__(execution_context_data, log_manager) self._required_resource_keys = get_required_resource_keys_for_step( step, execution_context_data.execution_plan, execution_context_data.intermediate_storage_def, ) self._resources = self._execution_context_data.scoped_resources_builder.build( self._required_resource_keys) step_launcher_resources = [ resource for resource in self._resources if isinstance(resource, StepLauncher) ] self._step_launcher: Optional[StepLauncher] = None if len(step_launcher_resources) > 1: raise DagsterInvariantViolationError( "Multiple required resources for solid {solid_name} have inherit StepLauncher" "There should be at most one step launcher resource per solid." .format(solid_name=step.solid_handle.name)) elif len(step_launcher_resources) == 1: self._step_launcher = step_launcher_resources[0] self._log_manager = log_manager
def __init__( self, plan_data: PlanData, execution_data: ExecutionData, log_manager: DagsterLogManager, step: ExecutionStep, output_capture: Optional[Dict[StepOutputHandle, Any]], previous_attempt_count: int, ): from dagster.core.execution.resources_init import get_required_resource_keys_for_step super(StepExecutionContext, self).__init__( plan_data=plan_data, execution_data=execution_data, log_manager=log_manager, output_capture=output_capture, ) self._step = step self._required_resource_keys = get_required_resource_keys_for_step( plan_data.pipeline.get_definition(), step, plan_data.execution_plan, ) self._resources = execution_data.scoped_resources_builder.build( self._required_resource_keys ) self._previous_attempt_count = previous_attempt_count self._input_lineage: List[AssetLineageInfo] = [] resources_iter = cast(Iterable, self._resources) step_launcher_resources = [ resource for resource in resources_iter if isinstance(resource, StepLauncher) ] self._step_launcher: Optional[StepLauncher] = None if len(step_launcher_resources) > 1: raise DagsterInvariantViolationError( "Multiple required resources for {described_op} have inherited StepLauncher" "There should be at most one step launcher resource per {node_type}.".format( described_op=self.describe_op(), node_type=self.solid_def.node_type_str ) ) elif len(step_launcher_resources) == 1: self._step_launcher = step_launcher_resources[0] self._step_exception: Optional[BaseException] = None self._step_output_capture: Optional[Dict[StepOutputHandle, Any]] = None # Enable step output capture if there are any hooks which will receive them. # Expect in the future that hooks may control whether or not they get outputs, # but for now presence of any will cause output capture. if self.pipeline_def.get_all_hooks_for_handle(self.solid_handle): self._step_output_capture = {} self._output_metadata: Dict[str, Any] = {} self._seen_outputs: Dict[str, Union[str, Set[str]]] = {}
def __init__( self, plan_data: PlanData, execution_data: ExecutionData, log_manager: DagsterLogManager, step: ExecutionStep, output_capture: Optional[Dict[StepOutputHandle, Any]], previous_attempt_count: int, ): from dagster.core.execution.resources_init import get_required_resource_keys_for_step super(StepExecutionContext, self).__init__( plan_data=plan_data, execution_data=execution_data, log_manager=log_manager, output_capture=output_capture, ) self._step = step self._required_resource_keys = get_required_resource_keys_for_step( plan_data.pipeline.get_definition(), step, plan_data.execution_plan, execution_data.environment_config, execution_data.intermediate_storage_def, ) self._resources = execution_data.scoped_resources_builder.build( self._required_resource_keys ) self._previous_attempt_count = previous_attempt_count resources_iter = cast(Iterable, self._resources) step_launcher_resources = [ resource for resource in resources_iter if isinstance(resource, StepLauncher) ] self._step_launcher: Optional[StepLauncher] = None if len(step_launcher_resources) > 1: raise DagsterInvariantViolationError( "Multiple required resources for solid {solid_name} have inherit StepLauncher" "There should be at most one step launcher resource per solid.".format( solid_name=step.solid_handle.name ) ) elif len(step_launcher_resources) == 1: self._step_launcher = step_launcher_resources[0] self._step_exception = None self._step_output_capture: Dict[StepOutputHandle, Any] = {}
def __init__(self, pipeline_context_data, log_manager, step): from dagster.core.execution.plan.objects import ExecutionStep from dagster.core.execution.resources_init import get_required_resource_keys_for_step self._step = check.inst_param(step, 'step', ExecutionStep) super(SystemStepExecutionContext, self).__init__(pipeline_context_data, log_manager) self._required_resource_keys = get_required_resource_keys_for_step( step, pipeline_context_data.pipeline_def, pipeline_context_data.system_storage_def, ) self._resources = self._pipeline_context_data.scoped_resources_builder.build( self._required_resource_keys) self._log_manager = log_manager