def step_context_to_step_run_ref(
step_context: StepExecutionContext,
prior_attempts_count: int,
package_dir: Optional[str] = None,
) -> StepRunRef:
"""
Args:
step_context (StepExecutionContext): The step context.
prior_attempts_count (int): The number of times this time has been tried before in the same
pipeline run.
package_dir (Optional[str]): If set, the reconstruction file code pointer will be converted
to be relative a module pointer relative to the package root. This enables executing
steps in remote setups where the package containing the pipeline resides at a different
location on the filesystem in the remote environment than in the environment executing
the plan process.
Returns (StepRunRef):
A reference to the step.
"""
check.inst_param(step_context, "step_context", StepExecutionContext)
check.int_param(prior_attempts_count, "prior_attempts_count")
retry_mode = step_context.retry_mode
recon_pipeline = step_context.pipeline
if package_dir:
if isinstance(recon_pipeline, ReconstructablePipeline) and isinstance(
recon_pipeline.repository.pointer, FileCodePointer):
recon_pipeline = ReconstructablePipeline(
repository=ReconstructableRepository(
pointer=ModuleCodePointer(
_module_in_package_dir(
recon_pipeline.repository.pointer.python_file,
package_dir),
recon_pipeline.repository.pointer.fn_name,
working_directory=os.getcwd(),
),
container_image=recon_pipeline.repository.container_image,
executable_path=recon_pipeline.repository.executable_path,
entry_point=recon_pipeline.repository.entry_point,
),
pipeline_name=recon_pipeline.pipeline_name,
solids_to_execute=recon_pipeline.solids_to_execute,
)
parent_run_id = step_context.pipeline_run.parent_run_id
parent_run = step_context.instance.get_run_by_id(
parent_run_id) if parent_run_id else None
return StepRunRef(
run_config=step_context.run_config,
pipeline_run=step_context.pipeline_run,
run_id=step_context.pipeline_run.run_id,
step_key=step_context.step.key,
retry_mode=retry_mode,
recon_pipeline=recon_pipeline,
prior_attempts_count=prior_attempts_count,
known_state=step_context.execution_plan.known_state,
parent_run=parent_run,
)
def step_context_to_step_run_ref(step_context, prior_attempts_count, package_dir=None):
'''
Args:
step_context (SystemStepExecutionContext): The step context.
prior_attempts_count (int): The number of times this time has been tried before in the same
pipeline run.
package_dir (Optional[str]): If set, the execution target will be converted to be relative
to the package root. This enables executing steps in remote setups where the package
containing the pipeline resides at a different location on the filesystem in the remote
environment than in the environment executing the plan process.
Returns (StepRunRef):
A reference to the step.
'''
execution_target_handle = step_context.execution_target_handle
if package_dir:
execution_target_handle = execution_target_handle.to_module_name_based_handle(package_dir)
return StepRunRef(
environment_dict=step_context.environment_dict,
pipeline_run=step_context.pipeline_run,
run_id=step_context.pipeline_run.run_id,
step_key=step_context.step.key,
executor_config=step_context.executor_config,
execution_target_handle=execution_target_handle,
prior_attempts_count=prior_attempts_count,
)
def step_context_to_step_run_ref(step_context, prior_attempts_count, package_dir=None):
'''
Args:
step_context (SystemStepExecutionContext): The step context.
prior_attempts_count (int): The number of times this time has been tried before in the same
pipeline run.
package_dir (Optional[str]): If set, the reconstruction file code pointer will be converted
to be relative a module pointer relative to the package root. This enables executing
steps in remote setups where the package containing the pipeline resides at a different
location on the filesystem in the remote environment than in the environment executing
the plan process.
Returns (StepRunRef):
A reference to the step.
'''
recon_pipeline = step_context.pipeline
if package_dir:
if isinstance(recon_pipeline, ReconstructablePipeline) and isinstance(
recon_pipeline.pointer, FileCodePointer
):
recon_pipeline = ReconstructablePipeline(
pointer=ModuleCodePointer(
_module_in_package_dir(recon_pipeline.pointer.python_file, package_dir),
recon_pipeline.pointer.fn_name,
),
frozen_solid_subset=recon_pipeline.frozen_solid_subset,
)
elif isinstance(recon_pipeline, ReconstructablePipelineFromRepo) and isinstance(
recon_pipeline.repository.pointer, FileCodePointer
):
recon_pipeline = ReconstructablePipelineFromRepo(
repository=ReconstructableRepository(
pointer=ModuleCodePointer(
_module_in_package_dir(
recon_pipeline.repository.pointer.python_file, package_dir
),
recon_pipeline.repository.pointer.fn_name,
),
),
pipeline_name=recon_pipeline.pipeline_name,
frozen_solid_subset=recon_pipeline.frozen_solid_subset,
)
return StepRunRef(
environment_dict=step_context.environment_dict,
pipeline_run=step_context.pipeline_run,
run_id=step_context.pipeline_run.run_id,
step_key=step_context.step.key,
executor_config=step_context.executor_config,
recon_pipeline=recon_pipeline,
prior_attempts_count=prior_attempts_count,
)
def step_context_to_step_run_ref(step_context,
prior_attempts_count,
package_dir=None):
'''
Args:
step_context (SystemStepExecutionContext): The step context.
prior_attempts_count (int): The number of times this time has been tried before in the same
pipeline run.
package_dir (Optional[str]): If set, the reconstruction file code pointer will be converted
to be relative a module pointer relative to the package root. This enables executing
steps in remote setups where the package containing the pipeline resides at a different
location on the filesystem in the remote environment than in the environment executing
the plan process.
Returns (StepRunRef):
A reference to the step.
'''
check.inst_param(step_context, 'step_context', SystemStepExecutionContext)
# This is a bit goofy right now, as the relationship between
# step launcher and the executor is a little weird.
#
# As far as I can tell, the only goal of interacting with the executor here
# is for the step launcher to inherit the retry policy. The step launcher
# actually assumes that a retry policy exists (hence the invariants below).
# However this isn't a formal requirement, so this can break. E.g. right
# now using a dask executor and the step launcher would fail.
#
# Once migration is complete, I think we should add a retries property
# to the Executor abc and make it optional. Then change this to handle
# the "no retry" case
check.param_invariant(
hasattr(step_context.executor, 'retries'),
'step_context',
'Executor must have retries property',
)
check.param_invariant(
isinstance(step_context.executor.retries, Retries),
'step_context',
'Executor retries property must be of type Retries. Object was {}'.
format(step_context.executor),
)
check.int_param(prior_attempts_count, 'prior_attempts_count')
retries = step_context.executor.retries
recon_pipeline = step_context.pipeline
if package_dir:
if isinstance(recon_pipeline, ReconstructablePipeline) and isinstance(
recon_pipeline.repository.pointer, FileCodePointer):
recon_pipeline = ReconstructablePipeline(
repository=ReconstructableRepository(pointer=ModuleCodePointer(
_module_in_package_dir(
recon_pipeline.repository.pointer.python_file,
package_dir),
recon_pipeline.repository.pointer.fn_name,
), ),
pipeline_name=recon_pipeline.pipeline_name,
solids_to_execute=recon_pipeline.solids_to_execute,
)
return StepRunRef(
run_config=step_context.run_config,
pipeline_run=step_context.pipeline_run,
run_id=step_context.pipeline_run.run_id,
step_key=step_context.step.key,
retries=retries,
recon_pipeline=recon_pipeline,
prior_attempts_count=prior_attempts_count,
)