def execute_run(
pipeline: IPipeline,
pipeline_run: PipelineRun,
instance: DagsterInstance,
raise_on_error: bool = False,
) -> PipelineExecutionResult:
"""Executes an existing pipeline run synchronously.
Synchronous version of execute_run_iterator.
Args:
pipeline (IPipeline): The pipeline to execute.
pipeline_run (PipelineRun): The run to execute
instance (DagsterInstance): The instance in which the run has been created.
raise_on_error (Optional[bool]): Whether or not to raise exceptions when they occur.
Defaults to ``False``.
Returns:
PipelineExecutionResult: The result of the execution.
"""
if isinstance(pipeline, PipelineDefinition):
raise DagsterInvariantViolationError(
"execute_run requires an IPipeline but received a PipelineDefinition "
"directly instead. To support hand-off to other processes provide a "
"ReconstructablePipeline which can be done using reconstructable(). For in "
"process only execution you can use InMemoryPipeline.")
check.inst_param(pipeline, "pipeline", IPipeline)
check.inst_param(pipeline_run, "pipeline_run", PipelineRun)
check.inst_param(instance, "instance", DagsterInstance)
if pipeline_run.status == PipelineRunStatus.CANCELED:
message = "Not starting execution since the run was canceled before execution could start"
instance.report_engine_event(
message,
pipeline_run,
)
raise DagsterInvariantViolationError(message)
check.invariant(
pipeline_run.status == PipelineRunStatus.NOT_STARTED
or pipeline_run.status == PipelineRunStatus.STARTING,
desc="Pipeline run {} ({}) in state {}, expected NOT_STARTED or STARTING"
.format(pipeline_run.pipeline_name, pipeline_run.run_id,
pipeline_run.status),
)
pipeline_def = pipeline.get_definition()
if pipeline_run.solids_to_execute:
if isinstance(pipeline_def, PipelineSubsetDefinition):
check.invariant(
pipeline_run.solids_to_execute == pipeline.solids_to_execute,
"Cannot execute PipelineRun with solids_to_execute {solids_to_execute} that "
"conflicts with pipeline subset {pipeline_solids_to_execute}.".
format(
pipeline_solids_to_execute=str_format_set(
pipeline.solids_to_execute),
solids_to_execute=str_format_set(
pipeline_run.solids_to_execute),
),
)
else:
# when `execute_run` is directly called, the sub pipeline hasn't been created
# note that when we receive the solids to execute via PipelineRun, it won't support
# solid selection query syntax
pipeline = pipeline.subset_for_execution_from_existing_pipeline(
pipeline_run.solids_to_execute)
execution_plan = _get_execution_plan_from_run(pipeline, pipeline_run,
instance)
if is_memoized_run(pipeline_run.tags):
resolved_run_config = ResolvedRunConfig.build(
pipeline.get_definition(), pipeline_run.run_config,
pipeline_run.mode)
execution_plan = resolve_memoized_execution_plan(
execution_plan,
pipeline.get_definition(),
pipeline_run.run_config,
instance,
resolved_run_config,
)
output_capture: Optional[Dict[StepOutputHandle, Any]] = {}
_execute_run_iterable = ExecuteRunWithPlanIterable(
execution_plan=execution_plan,
iterator=pipeline_execution_iterator,
execution_context_manager=PlanOrchestrationContextManager(
context_event_generator=orchestration_context_event_generator,
pipeline=pipeline,
execution_plan=execution_plan,
pipeline_run=pipeline_run,
instance=instance,
run_config=pipeline_run.run_config,
raise_on_error=raise_on_error,
executor_defs=None,
output_capture=output_capture,
),
)
event_list = list(_execute_run_iterable)
return PipelineExecutionResult(
pipeline.get_definition(),
pipeline_run.run_id,
event_list,
lambda: scoped_pipeline_context(
execution_plan,
pipeline,
pipeline_run.run_config,
pipeline_run,
instance,
),
output_capture=output_capture,
)
def create_valid_pipeline_run(graphene_info, external_pipeline,
execution_params):
ensure_valid_config(external_pipeline, execution_params.mode,
execution_params.run_config)
step_keys_to_execute = compute_step_keys_to_execute(
graphene_info, external_pipeline, execution_params)
external_execution_plan = get_external_execution_plan_or_raise(
graphene_info=graphene_info,
external_pipeline=external_pipeline,
mode=execution_params.mode,
run_config=execution_params.run_config,
step_keys_to_execute=step_keys_to_execute,
)
tags = merge_dicts(external_pipeline.tags,
execution_params.execution_metadata.tags)
pipeline_run = graphene_info.context.instance.create_run(
pipeline_snapshot=external_pipeline.pipeline_snapshot,
execution_plan_snapshot=external_execution_plan.
execution_plan_snapshot,
parent_pipeline_snapshot=external_pipeline.parent_pipeline_snapshot,
pipeline_name=execution_params.selector.pipeline_name,
run_id=execution_params.execution_metadata.run_id
if execution_params.execution_metadata.run_id else make_new_run_id(),
solids_to_execute=frozenset(execution_params.selector.solid_selection)
if execution_params.selector.solid_selection else None,
run_config=execution_params.run_config,
mode=execution_params.mode,
step_keys_to_execute=step_keys_to_execute,
tags=tags,
root_run_id=execution_params.execution_metadata.root_run_id,
parent_run_id=execution_params.execution_metadata.parent_run_id,
status=PipelineRunStatus.NOT_STARTED,
external_pipeline_origin=external_pipeline.get_external_origin(),
)
# TODO: support memoized execution from dagit. https://github.com/dagster-io/dagster/issues/3322
if is_memoized_run(tags):
graphene_info.context.instance.report_engine_event(
'Tag "{tag}" was found when initializing pipeline run, however, memoized '
"execution is only supported from the dagster CLI. This pipeline will run, but "
"outputs from previous executions will be ignored. "
"In order to execute this pipeline using memoization, provide the "
'"{tag}" tag to the `dagster pipeline execute` CLI. The CLI is documented at '
"the provided link.".format(tag=MEMOIZED_RUN_TAG),
pipeline_run,
EngineEventData([
EventMetadataEntry.url(
"https://docs.dagster.io/_apidocs/cli#dagster-pipeline-execute",
label="dagster_pipeline_execute_docs_url",
description=
"In order to execute this pipeline using memoization, provide the "
'"{tag}" tag to the `dagster pipeline execute` CLI. The CLI is documented at '
"the provided link.".format(tag=MEMOIZED_RUN_TAG),
)
]),
)
return pipeline_run
def execute_run(
pipeline: IPipeline,
pipeline_run: PipelineRun,
instance: DagsterInstance,
raise_on_error: bool = False,
) -> PipelineExecutionResult:
"""Executes an existing pipeline run synchronously.
Synchronous version of execute_run_iterator.
Args:
pipeline (IPipeline): The pipeline to execute.
pipeline_run (PipelineRun): The run to execute
instance (DagsterInstance): The instance in which the run has been created.
raise_on_error (Optional[bool]): Whether or not to raise exceptions when they occur.
Defaults to ``False``.
Returns:
PipelineExecutionResult: The result of the execution.
"""
if isinstance(pipeline, PipelineDefinition):
raise DagsterInvariantViolationError(
"execute_run requires an IPipeline but received a PipelineDefinition "
"directly instead. To support hand-off to other processes provide a "
"ReconstructablePipeline which can be done using reconstructable(). For in "
"process only execution you can use InMemoryPipeline.")
check.inst_param(pipeline, "pipeline", IPipeline)
check.inst_param(pipeline_run, "pipeline_run", PipelineRun)
check.inst_param(instance, "instance", DagsterInstance)
if pipeline_run.status == PipelineRunStatus.CANCELED:
message = "Not starting execution since the run was canceled before execution could start"
instance.report_engine_event(
message,
pipeline_run,
)
raise DagsterInvariantViolationError(message)
check.invariant(
pipeline_run.status == PipelineRunStatus.NOT_STARTED
or pipeline_run.status == PipelineRunStatus.STARTING,
desc="Pipeline run {} ({}) in state {}, expected NOT_STARTED or STARTING"
.format(pipeline_run.pipeline_name, pipeline_run.run_id,
pipeline_run.status),
)
pipeline_def = pipeline.get_definition()
if pipeline_run.solids_to_execute:
if isinstance(pipeline_def, PipelineSubsetDefinition):
check.invariant(
pipeline_run.solids_to_execute == pipeline.solids_to_execute,
"Cannot execute PipelineRun with solids_to_execute {solids_to_execute} that "
"conflicts with pipeline subset {pipeline_solids_to_execute}.".
format(
pipeline_solids_to_execute=str_format_set(
pipeline.solids_to_execute),
solids_to_execute=str_format_set(
pipeline_run.solids_to_execute),
),
)
else:
# when `execute_run` is directly called, the sub pipeline hasn't been created
# note that when we receive the solids to execute via PipelineRun, it won't support
# solid selection query syntax
pipeline = pipeline.subset_for_execution_from_existing_pipeline(
pipeline_run.solids_to_execute)
execution_plan = create_execution_plan(
pipeline,
run_config=pipeline_run.run_config,
mode=pipeline_run.mode,
step_keys_to_execute=pipeline_run.step_keys_to_execute,
)
if is_memoized_run(pipeline_run.tags):
execution_plan = resolve_memoized_execution_plan(execution_plan)
_execute_run_iterable = _ExecuteRunWithPlanIterable(
execution_plan=execution_plan,
iterator=_pipeline_execution_iterator,
execution_context_manager=PipelineExecutionContextManager(
execution_plan=execution_plan,
pipeline_run=pipeline_run,
instance=instance,
run_config=pipeline_run.run_config,
raise_on_error=raise_on_error,
),
)
event_list = list(_execute_run_iterable)
pipeline_context = _execute_run_iterable.pipeline_context
# workaround for mem_io_manager to work in reconstruct_context, e.g. result.result_for_solid
# in-memory values dict will get lost when the resource is re-initiated in reconstruct_context
# so instead of re-initiating every single resource, we pass the resource instances to
# reconstruct_context directly to avoid re-building from resource def.
resource_instances_to_override = {}
if pipeline_context: # None if we have a pipeline failure
for (
key,
resource_instance,
) in pipeline_context.scoped_resources_builder.resource_instance_dict.items(
):
if isinstance(resource_instance, InMemoryIOManager):
resource_instances_to_override[key] = resource_instance
return PipelineExecutionResult(
pipeline.get_definition(),
pipeline_run.run_id,
event_list,
lambda hardcoded_resources_arg: scoped_pipeline_context(
execution_plan,
pipeline_run.run_config,
pipeline_run,
instance,
intermediate_storage=pipeline_context.intermediate_storage,
resource_instances_to_override=hardcoded_resources_arg,
),
resource_instances_to_override=resource_instances_to_override,
)
def execute_run(pipeline, pipeline_run, instance, raise_on_error=False):
"""Executes an existing pipeline run synchronously.
Synchronous version of execute_run_iterator.
Args:
pipeline (IPipeline): The pipeline to execute.
pipeline_run (PipelineRun): The run to execute
instance (DagsterInstance): The instance in which the run has been created.
raise_on_error (Optional[bool]): Whether or not to raise exceptions when they occur.
Defaults to ``False``.
Returns:
PipelineExecutionResult: The result of the execution.
"""
if isinstance(pipeline, PipelineDefinition):
raise DagsterInvariantViolationError(
"execute_run requires an IPipeline but received a PipelineDefinition "
"directly instead. To support hand-off to other processes provide a "
"ReconstructablePipeline which can be done using reconstructable(). For in "
"process only execution you can use InMemoryPipeline.")
check.inst_param(pipeline, "pipeline", IPipeline)
check.inst_param(pipeline_run, "pipeline_run", PipelineRun)
check.inst_param(instance, "instance", DagsterInstance)
check.invariant(
pipeline_run.status == PipelineRunStatus.NOT_STARTED,
desc=
"Pipeline run {} ({}) in state {}, expected PipelineRunStatus.NOT_STARTED"
.format(pipeline_run.pipeline_name, pipeline_run.run_id,
pipeline_run.status),
)
pipeline_def = pipeline.get_definition()
if pipeline_run.solids_to_execute:
if isinstance(pipeline_def, PipelineSubsetDefinition):
check.invariant(
pipeline_run.solids_to_execute == pipeline.solids_to_execute,
"Cannot execute PipelineRun with solids_to_execute {solids_to_execute} that "
"conflicts with pipeline subset {pipeline_solids_to_execute}.".
format(
pipeline_solids_to_execute=str_format_set(
pipeline.solids_to_execute),
solids_to_execute=str_format_set(
pipeline_run.solids_to_execute),
),
)
else:
# when `execute_run` is directly called, the sub pipeline hasn't been created
# note that when we receive the solids to execute via PipelineRun, it won't support
# solid selection query syntax
pipeline = pipeline.subset_for_execution_from_existing_pipeline(
pipeline_run.solids_to_execute)
execution_plan = create_execution_plan(
pipeline,
run_config=pipeline_run.run_config,
mode=pipeline_run.mode,
step_keys_to_execute=pipeline_run.step_keys_to_execute,
)
if is_memoized_run(pipeline_run.tags):
execution_plan = instance.resolve_memoized_execution_plan(
execution_plan,
run_config=pipeline_run.run_config,
mode=pipeline_run.mode)
_execute_run_iterable = _ExecuteRunWithPlanIterable(
execution_plan=execution_plan,
iterator=_pipeline_execution_iterator,
execution_context_manager=PipelineExecutionContextManager(
execution_plan=execution_plan,
pipeline_run=pipeline_run,
instance=instance,
run_config=pipeline_run.run_config,
raise_on_error=raise_on_error,
),
)
event_list = list(_execute_run_iterable)
pipeline_context = _execute_run_iterable.pipeline_context
return PipelineExecutionResult(
pipeline.get_definition(),
pipeline_run.run_id,
event_list,
lambda: scoped_pipeline_context(
execution_plan,
pipeline_run.run_config,
pipeline_run,
instance,
intermediate_storage=pipeline_context.intermediate_storage,
system_storage_data=SystemStorageData(
intermediate_storage=pipeline_context.intermediate_storage,
file_manager=pipeline_context.file_manager,
),
),
)
