def _create_external_pipeline_run( instance, repo_location, external_repo, external_pipeline, run_config, mode, preset, tags, solid_selection, run_id, ): check.inst_param(instance, "instance", DagsterInstance) check.inst_param(repo_location, "repo_location", RepositoryLocation) check.inst_param(external_repo, "external_repo", ExternalRepository) check.inst_param(external_pipeline, "external_pipeline", ExternalPipeline) check.opt_dict_param(run_config, "run_config") check.opt_str_param(mode, "mode") check.opt_str_param(preset, "preset") check.opt_dict_param(tags, "tags", key_type=str) check.opt_list_param(solid_selection, "solid_selection", of_type=str) check.opt_str_param(run_id, "run_id") run_config, mode, tags, solid_selection = _check_execute_external_pipeline_args( external_pipeline, run_config, mode, preset, tags, solid_selection, ) pipeline_name = external_pipeline.name pipeline_selector = PipelineSelector( location_name=repo_location.name, repository_name=external_repo.name, pipeline_name=pipeline_name, solid_selection=solid_selection, ) subset_pipeline_result = repo_location.get_subset_external_pipeline_result( pipeline_selector) if subset_pipeline_result.success == False: raise DagsterLaunchFailedError( "Failed to load external pipeline subset: {error_message}".format( error_message=subset_pipeline_result.error.message), serializable_error_info=subset_pipeline_result.error, ) external_pipeline_subset = ExternalPipeline( subset_pipeline_result.external_pipeline_data, external_repo.handle, ) pipeline_mode = mode or external_pipeline_subset.get_default_mode_name() external_execution_plan = repo_location.get_external_execution_plan( external_pipeline_subset, run_config, pipeline_mode, step_keys_to_execute=None, ) if isinstance(external_execution_plan, ExecutionPlanSnapshotErrorData): raise DagsterLaunchFailedError( "Failed to load external execution plan", serializable_error_info=external_execution_plan.error, ) else: execution_plan_snapshot = external_execution_plan.execution_plan_snapshot return instance.create_run( pipeline_name=pipeline_name, run_id=run_id, run_config=run_config, mode=pipeline_mode, solids_to_execute=external_pipeline_subset.solids_to_execute, step_keys_to_execute=None, solid_selection=solid_selection, status=None, root_run_id=None, parent_run_id=None, tags=tags, pipeline_snapshot=external_pipeline_subset.pipeline_snapshot, execution_plan_snapshot=execution_plan_snapshot, parent_pipeline_snapshot=external_pipeline_subset. parent_pipeline_snapshot, external_pipeline_origin=external_pipeline_subset.get_external_origin( ), )
def _create_external_pipeline_run( instance, repo_location, external_repo, external_pipeline, run_config, mode, preset, tags, solid_selection, run_id, ): check.inst_param(instance, "instance", DagsterInstance) check.inst_param(repo_location, "repo_location", RepositoryLocation) check.inst_param(external_repo, "external_repo", ExternalRepository) check.inst_param(external_pipeline, "external_pipeline", ExternalPipeline) check.opt_dict_param(run_config, "run_config") check.opt_str_param(mode, "mode") check.opt_str_param(preset, "preset") check.opt_dict_param(tags, "tags", key_type=str) check.opt_list_param(solid_selection, "solid_selection", of_type=str) check.opt_str_param(run_id, "run_id") run_config, mode, tags, solid_selection = _check_execute_external_pipeline_args( external_pipeline, run_config, mode, preset, tags, solid_selection, ) pipeline_name = external_pipeline.name pipeline_selector = PipelineSelector( location_name=repo_location.name, repository_name=external_repo.name, pipeline_name=pipeline_name, solid_selection=solid_selection, ) subset_pipeline_result = repo_location.get_subset_external_pipeline_result(pipeline_selector) external_pipeline_subset = ExternalPipeline( subset_pipeline_result.external_pipeline_data, external_repo.handle, ) pipeline_mode = mode or external_pipeline_subset.get_default_mode_name() external_execution_plan = repo_location.get_external_execution_plan( external_pipeline_subset, run_config, pipeline_mode, step_keys_to_execute=None, known_state=None, ) execution_plan_snapshot = external_execution_plan.execution_plan_snapshot return instance.create_run( pipeline_name=pipeline_name, run_id=run_id, run_config=run_config, mode=pipeline_mode, solids_to_execute=external_pipeline_subset.solids_to_execute, step_keys_to_execute=None, solid_selection=solid_selection, status=None, root_run_id=None, parent_run_id=None, tags=tags, pipeline_snapshot=external_pipeline_subset.pipeline_snapshot, execution_plan_snapshot=execution_plan_snapshot, parent_pipeline_snapshot=external_pipeline_subset.parent_pipeline_snapshot, external_pipeline_origin=external_pipeline_subset.get_external_origin(), pipeline_code_origin=external_pipeline.get_python_origin(), )
def _create_external_pipeline_run( instance: DagsterInstance, repo_location: RepositoryLocation, external_repo: ExternalRepository, external_pipeline: ExternalPipeline, run_config: Dict[str, object], mode: Optional[str], preset: Optional[str], tags: Optional[Mapping[str, object]], solid_selection: Optional[List[str]], run_id: Optional[str], ): check.inst_param(instance, "instance", DagsterInstance) check.inst_param(repo_location, "repo_location", RepositoryLocation) check.inst_param(external_repo, "external_repo", ExternalRepository) check.inst_param(external_pipeline, "external_pipeline", ExternalPipeline) check.opt_dict_param(run_config, "run_config", key_type=str) check.opt_str_param(mode, "mode") check.opt_str_param(preset, "preset") check.opt_dict_param(tags, "tags", key_type=str) check.opt_list_param(solid_selection, "solid_selection", of_type=str) check.opt_str_param(run_id, "run_id") run_config, mode, tags, solid_selection = _check_execute_external_pipeline_args( external_pipeline, run_config, mode, preset, tags, solid_selection, ) pipeline_name = external_pipeline.name pipeline_selector = PipelineSelector( location_name=repo_location.name, repository_name=external_repo.name, pipeline_name=pipeline_name, solid_selection=solid_selection, ) external_pipeline = repo_location.get_external_pipeline(pipeline_selector) pipeline_mode = mode or external_pipeline.get_default_mode_name() external_execution_plan = repo_location.get_external_execution_plan( external_pipeline, run_config, pipeline_mode, step_keys_to_execute=None, known_state=None, instance=instance, ) execution_plan_snapshot = external_execution_plan.execution_plan_snapshot return instance.create_run( pipeline_name=pipeline_name, run_id=run_id, run_config=run_config, mode=pipeline_mode, solids_to_execute=external_pipeline.solids_to_execute, step_keys_to_execute=execution_plan_snapshot.step_keys_to_execute, solid_selection=solid_selection, status=None, root_run_id=None, parent_run_id=None, tags=tags, pipeline_snapshot=external_pipeline.pipeline_snapshot, execution_plan_snapshot=execution_plan_snapshot, parent_pipeline_snapshot=external_pipeline.parent_pipeline_snapshot, external_pipeline_origin=external_pipeline.get_external_origin(), pipeline_code_origin=external_pipeline.get_python_origin(), )