def _create_external_pipeline_run(
instance,
repo_location,
external_repo,
external_pipeline,
run_config,
mode,
preset,
tags,
solid_selection,
run_id,
):
check.inst_param(instance, "instance", DagsterInstance)
check.inst_param(repo_location, "repo_location", RepositoryLocation)
check.inst_param(external_repo, "external_repo", ExternalRepository)
check.inst_param(external_pipeline, "external_pipeline", ExternalPipeline)
check.opt_dict_param(run_config, "run_config")
check.opt_str_param(mode, "mode")
check.opt_str_param(preset, "preset")
check.opt_dict_param(tags, "tags", key_type=str)
check.opt_list_param(solid_selection, "solid_selection", of_type=str)
check.opt_str_param(run_id, "run_id")
run_config, mode, tags, solid_selection = _check_execute_external_pipeline_args(
external_pipeline,
run_config,
mode,
preset,
tags,
solid_selection,
)
pipeline_name = external_pipeline.name
pipeline_selector = PipelineSelector(
location_name=repo_location.name,
repository_name=external_repo.name,
pipeline_name=pipeline_name,
solid_selection=solid_selection,
)
subset_pipeline_result = repo_location.get_subset_external_pipeline_result(
pipeline_selector)
if subset_pipeline_result.success == False:
raise DagsterLaunchFailedError(
"Failed to load external pipeline subset: {error_message}".format(
error_message=subset_pipeline_result.error.message),
serializable_error_info=subset_pipeline_result.error,
)
external_pipeline_subset = ExternalPipeline(
subset_pipeline_result.external_pipeline_data,
external_repo.handle,
)
pipeline_mode = mode or external_pipeline_subset.get_default_mode_name()
external_execution_plan = repo_location.get_external_execution_plan(
external_pipeline_subset,
run_config,
pipeline_mode,
step_keys_to_execute=None,
)
if isinstance(external_execution_plan, ExecutionPlanSnapshotErrorData):
raise DagsterLaunchFailedError(
"Failed to load external execution plan",
serializable_error_info=external_execution_plan.error,
)
else:
execution_plan_snapshot = external_execution_plan.execution_plan_snapshot
return instance.create_run(
pipeline_name=pipeline_name,
run_id=run_id,
run_config=run_config,
mode=pipeline_mode,
solids_to_execute=external_pipeline_subset.solids_to_execute,
step_keys_to_execute=None,
solid_selection=solid_selection,
status=None,
root_run_id=None,
parent_run_id=None,
tags=tags,
pipeline_snapshot=external_pipeline_subset.pipeline_snapshot,
execution_plan_snapshot=execution_plan_snapshot,
parent_pipeline_snapshot=external_pipeline_subset.
parent_pipeline_snapshot,
external_pipeline_origin=external_pipeline_subset.get_external_origin(
),
)
def _create_external_pipeline_run(
instance,
repo_location,
external_repo,
external_pipeline,
run_config,
mode,
preset,
tags,
solid_selection,
run_id,
):
check.inst_param(instance, "instance", DagsterInstance)
check.inst_param(repo_location, "repo_location", RepositoryLocation)
check.inst_param(external_repo, "external_repo", ExternalRepository)
check.inst_param(external_pipeline, "external_pipeline", ExternalPipeline)
check.opt_dict_param(run_config, "run_config")
check.opt_str_param(mode, "mode")
check.opt_str_param(preset, "preset")
check.opt_dict_param(tags, "tags", key_type=str)
check.opt_list_param(solid_selection, "solid_selection", of_type=str)
check.opt_str_param(run_id, "run_id")
run_config, mode, tags, solid_selection = _check_execute_external_pipeline_args(
external_pipeline,
run_config,
mode,
preset,
tags,
solid_selection,
)
pipeline_name = external_pipeline.name
pipeline_selector = PipelineSelector(
location_name=repo_location.name,
repository_name=external_repo.name,
pipeline_name=pipeline_name,
solid_selection=solid_selection,
)
subset_pipeline_result = repo_location.get_subset_external_pipeline_result(pipeline_selector)
external_pipeline_subset = ExternalPipeline(
subset_pipeline_result.external_pipeline_data,
external_repo.handle,
)
pipeline_mode = mode or external_pipeline_subset.get_default_mode_name()
external_execution_plan = repo_location.get_external_execution_plan(
external_pipeline_subset,
run_config,
pipeline_mode,
step_keys_to_execute=None,
known_state=None,
)
execution_plan_snapshot = external_execution_plan.execution_plan_snapshot
return instance.create_run(
pipeline_name=pipeline_name,
run_id=run_id,
run_config=run_config,
mode=pipeline_mode,
solids_to_execute=external_pipeline_subset.solids_to_execute,
step_keys_to_execute=None,
solid_selection=solid_selection,
status=None,
root_run_id=None,
parent_run_id=None,
tags=tags,
pipeline_snapshot=external_pipeline_subset.pipeline_snapshot,
execution_plan_snapshot=execution_plan_snapshot,
parent_pipeline_snapshot=external_pipeline_subset.parent_pipeline_snapshot,
external_pipeline_origin=external_pipeline_subset.get_external_origin(),
pipeline_code_origin=external_pipeline.get_python_origin(),
)
def _create_external_pipeline_run(
instance: DagsterInstance,
repo_location: RepositoryLocation,
external_repo: ExternalRepository,
external_pipeline: ExternalPipeline,
run_config: Dict[str, object],
mode: Optional[str],
preset: Optional[str],
tags: Optional[Mapping[str, object]],
solid_selection: Optional[List[str]],
run_id: Optional[str],
):
check.inst_param(instance, "instance", DagsterInstance)
check.inst_param(repo_location, "repo_location", RepositoryLocation)
check.inst_param(external_repo, "external_repo", ExternalRepository)
check.inst_param(external_pipeline, "external_pipeline", ExternalPipeline)
check.opt_dict_param(run_config, "run_config", key_type=str)
check.opt_str_param(mode, "mode")
check.opt_str_param(preset, "preset")
check.opt_dict_param(tags, "tags", key_type=str)
check.opt_list_param(solid_selection, "solid_selection", of_type=str)
check.opt_str_param(run_id, "run_id")
run_config, mode, tags, solid_selection = _check_execute_external_pipeline_args(
external_pipeline,
run_config,
mode,
preset,
tags,
solid_selection,
)
pipeline_name = external_pipeline.name
pipeline_selector = PipelineSelector(
location_name=repo_location.name,
repository_name=external_repo.name,
pipeline_name=pipeline_name,
solid_selection=solid_selection,
)
external_pipeline = repo_location.get_external_pipeline(pipeline_selector)
pipeline_mode = mode or external_pipeline.get_default_mode_name()
external_execution_plan = repo_location.get_external_execution_plan(
external_pipeline,
run_config,
pipeline_mode,
step_keys_to_execute=None,
known_state=None,
instance=instance,
)
execution_plan_snapshot = external_execution_plan.execution_plan_snapshot
return instance.create_run(
pipeline_name=pipeline_name,
run_id=run_id,
run_config=run_config,
mode=pipeline_mode,
solids_to_execute=external_pipeline.solids_to_execute,
step_keys_to_execute=execution_plan_snapshot.step_keys_to_execute,
solid_selection=solid_selection,
status=None,
root_run_id=None,
parent_run_id=None,
tags=tags,
pipeline_snapshot=external_pipeline.pipeline_snapshot,
execution_plan_snapshot=execution_plan_snapshot,
parent_pipeline_snapshot=external_pipeline.parent_pipeline_snapshot,
external_pipeline_origin=external_pipeline.get_external_origin(),
pipeline_code_origin=external_pipeline.get_python_origin(),
)