def step_run_ref_to_step_context(step_run_ref): pipeline_def = step_run_ref.recon_pipeline.get_definition( ).get_pipeline_subset_def(step_run_ref.pipeline_run.solids_to_execute) execution_plan = create_execution_plan( pipeline_def, step_run_ref.environment_dict, mode=step_run_ref.pipeline_run.mode).build_subset_plan( [step_run_ref.step_key]) initialization_manager = pipeline_initialization_manager( execution_plan, step_run_ref.environment_dict, step_run_ref.pipeline_run, DagsterInstance.ephemeral(), ) for _ in initialization_manager.generate_setup_events(): pass pipeline_context = initialization_manager.get_object() retries = step_run_ref.executor_config.retries.for_inner_plan() active_execution = execution_plan.start(retries=retries) step = active_execution.get_next_step() pipeline_context = initialization_manager.get_object() return pipeline_context.for_step(step)
def initialize_step_context(scratch_dir): pipeline_run = PipelineRun( pipeline_name='foo_pipeline', run_id=str(uuid.uuid4()), run_config=make_run_config(scratch_dir, 'external'), mode='external', ) plan = create_execution_plan(reconstructable(define_basic_pipeline), pipeline_run.run_config, mode='external') initialization_manager = pipeline_initialization_manager( plan, pipeline_run.run_config, pipeline_run, DagsterInstance.ephemeral(), ) for _ in initialization_manager.generate_setup_events(): pass pipeline_context = initialization_manager.get_object() active_execution = plan.start(retries=Retries(RetryMode.DISABLED)) step = active_execution.get_next_step() step_context = pipeline_context.for_step(step) return step_context
def initialize_step_context(scratch_dir): pipeline_def = ExecutionTargetHandle.for_pipeline_fn( define_basic_pipeline).build_pipeline_definition() pipeline_run = PipelineRun( pipeline_name='foo_pipeline', run_id=str(uuid.uuid4()), environment_dict=make_environment_dict(scratch_dir, 'external'), mode='external', ) plan = create_execution_plan(pipeline_def, pipeline_run.environment_dict, mode='external') initialization_manager = pipeline_initialization_manager( plan, pipeline_run.environment_dict, pipeline_run, DagsterInstance.ephemeral(), ) for _ in initialization_manager.generate_setup_events(): pass pipeline_context = initialization_manager.get_object() active_execution = plan.start(retries=Retries(RetryMode.DISABLED)) step = active_execution.get_next_step() step_context = pipeline_context.for_step(step) return step_context