def initialize_step_context(scratch_dir, instance):
pipeline_run = PipelineRun(
pipeline_name="foo_pipeline",
run_id=str(uuid.uuid4()),
run_config=make_run_config(scratch_dir, "external"),
mode="external",
)
recon_pipeline = reconstructable(define_basic_pipeline)
plan = create_execution_plan(recon_pipeline,
pipeline_run.run_config,
mode="external")
initialization_manager = PlanExecutionContextManager(
pipeline=recon_pipeline,
execution_plan=plan,
run_config=pipeline_run.run_config,
pipeline_run=pipeline_run,
instance=instance,
retry_mode=RetryMode.DISABLED,
)
for _ in initialization_manager.prepare_context():
pass
pipeline_context = initialization_manager.get_context()
step_context = pipeline_context.for_step(
plan.get_step_by_key("return_two"))
return step_context
def step_run_ref_to_step_context(
step_run_ref: StepRunRef, instance: DagsterInstance
) -> StepExecutionContext:
check.inst_param(instance, "instance", DagsterInstance)
pipeline = step_run_ref.recon_pipeline.subset_for_execution_from_existing_pipeline(
step_run_ref.pipeline_run.solids_to_execute
)
execution_plan = create_execution_plan(
pipeline,
step_run_ref.run_config,
mode=step_run_ref.pipeline_run.mode,
step_keys_to_execute=[step_run_ref.step_key],
)
initialization_manager = PlanExecutionContextManager(
retry_mode=step_run_ref.retry_mode.for_inner_plan(),
pipeline=pipeline,
execution_plan=execution_plan,
run_config=step_run_ref.run_config,
pipeline_run=step_run_ref.pipeline_run,
instance=instance,
)
for _ in initialization_manager.prepare_context():
pass
execution_context = initialization_manager.get_context()
execution_step = cast("ExecutionStep", execution_plan.get_step_by_key(step_run_ref.step_key))
step_execution_context = execution_context.for_step(execution_step)
# Since for_step is abstract for IPlanContext, its return type is IStepContext.
# Since we are launching from a PlanExecutionContext, the type will always be
# StepExecutionContext.
step_execution_context = cast(StepExecutionContext, step_execution_context)
return step_execution_context
def step_run_ref_to_step_context(
step_run_ref: StepRunRef, instance: DagsterInstance
) -> SystemStepExecutionContext:
check.inst_param(instance, "instance", DagsterInstance)
pipeline = step_run_ref.recon_pipeline.subset_for_execution_from_existing_pipeline(
step_run_ref.pipeline_run.solids_to_execute
)
execution_plan = create_execution_plan(
pipeline,
step_run_ref.run_config,
mode=step_run_ref.pipeline_run.mode,
step_keys_to_execute=[step_run_ref.step_key],
)
initialization_manager = PlanExecutionContextManager(
retry_mode=step_run_ref.retry_mode.for_inner_plan(),
pipeline=pipeline,
execution_plan=execution_plan,
run_config=step_run_ref.run_config,
pipeline_run=step_run_ref.pipeline_run,
instance=instance,
)
for _ in initialization_manager.prepare_context():
pass
execution_context = initialization_manager.get_context()
return execution_context.for_step(execution_plan.get_step_by_key(step_run_ref.step_key))
def step_run_ref_to_step_context(step_run_ref):
pipeline = step_run_ref.recon_pipeline.subset_for_execution_from_existing_pipeline(
step_run_ref.pipeline_run.solids_to_execute
)
execution_plan = create_execution_plan(
pipeline, step_run_ref.run_config, mode=step_run_ref.pipeline_run.mode
).build_subset_plan([step_run_ref.step_key])
retries = step_run_ref.retries.for_inner_plan()
initialization_manager = PlanExecutionContextManager(
retries,
execution_plan,
step_run_ref.run_config,
step_run_ref.pipeline_run,
DagsterInstance.ephemeral(),
)
for _ in initialization_manager.prepare_context():
pass
execution_context = initialization_manager.get_context()
active_execution = execution_plan.start(retries=retries)
step = active_execution.get_next_step()
return execution_context.for_step(step)
def step_run_ref_to_step_context(step_run_ref, instance):
check.inst_param(instance, "instance", DagsterInstance)
pipeline = step_run_ref.recon_pipeline.subset_for_execution_from_existing_pipeline(
step_run_ref.pipeline_run.solids_to_execute)
execution_plan = create_execution_plan(
pipeline, step_run_ref.run_config,
mode=step_run_ref.pipeline_run.mode).build_subset_plan(
[step_run_ref.step_key])
retries = step_run_ref.retries.for_inner_plan()
initialization_manager = PlanExecutionContextManager(
retries,
execution_plan,
step_run_ref.run_config,
step_run_ref.pipeline_run,
instance,
)
for _ in initialization_manager.prepare_context():
pass
execution_context = initialization_manager.get_context()
return execution_context.for_step(
execution_plan.get_step_by_key(step_run_ref.step_key))