def create_execution_plan(pipeline,
environment_dict=None,
mode=None,
step_keys_to_execute=None):
# backcompat
if isinstance(pipeline, PipelineDefinition):
pipeline = InMemoryExecutablePipeline(pipeline)
check.inst_param(pipeline, 'pipeline', ExecutablePipeline)
pipeline_def = pipeline.get_definition()
environment_dict = check.opt_dict_param(environment_dict,
'environment_dict',
key_type=str)
mode = check.opt_str_param(mode,
'mode',
default=pipeline_def.get_default_mode_name())
check.opt_list_param(step_keys_to_execute,
'step_keys_to_execute',
of_type=str)
environment_config = EnvironmentConfig.build(pipeline_def,
environment_dict,
mode=mode)
return ExecutionPlan.build(pipeline,
environment_config,
mode=mode,
step_keys_to_execute=step_keys_to_execute)
def yield_empty_pipeline_context(run_id=None, instance=None):
pipeline = InMemoryExecutablePipeline(PipelineDefinition([]))
pipeline_def = pipeline.get_definition()
instance = check.opt_inst_param(
instance, 'instance', DagsterInstance, default=DagsterInstance.ephemeral()
)
execution_plan = create_execution_plan(pipeline)
pipeline_run = instance.create_run(
pipeline_name='<empty>',
run_id=run_id,
run_config=None,
mode=None,
solids_to_execute=None,
step_keys_to_execute=None,
status=None,
tags=None,
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=pipeline_def.get_pipeline_snapshot(),
execution_plan_snapshot=snapshot_from_execution_plan(
execution_plan, pipeline_def.get_pipeline_snapshot_id()
),
parent_pipeline_snapshot=pipeline_def.get_parent_pipeline_snapshot(),
)
with scoped_pipeline_context(execution_plan, {}, pipeline_run, instance) as context:
yield context
def _check_pipeline(pipeline):
# backcompat
if isinstance(pipeline, PipelineDefinition):
pipeline = InMemoryExecutablePipeline(pipeline)
check.inst_param(pipeline, 'pipeline', ExecutablePipeline)
pipeline_def = pipeline.get_definition()
return pipeline, pipeline_def
def _check_execute_pipeline_args(fn_name, pipeline, environment_dict, mode,
preset, tags, run_config, instance):
# backcompat
if isinstance(pipeline, PipelineDefinition):
pipeline = InMemoryExecutablePipeline(pipeline)
check.inst_param(pipeline, 'pipeline', ExecutablePipeline)
pipeline_def = pipeline.get_definition()
environment_dict = check.opt_dict_param(environment_dict,
'environment_dict')
check.opt_str_param(mode, 'mode')
check.opt_str_param(preset, 'preset')
check.invariant(
not (mode is not None and preset is not None),
'You may set only one of `mode` (got {mode}) or `preset` (got {preset}).'
.format(mode=mode, preset=preset),
)
tags = check.opt_dict_param(tags, 'tags', key_type=str)
run_config = check.opt_inst_param(run_config,
'run_config',
RunConfig,
default=RunConfig())
if preset is not None:
pipeline_preset = pipeline_def.get_preset(preset)
check.invariant(
run_config.mode is None or pipeline_preset.mode == run_config.mode,
'The mode set in preset \'{preset}\' (\'{preset_mode}\') does not agree with the mode '
'set in the `run_config` (\'{run_config_mode}\')'.format(
preset=preset,
preset_mode=pipeline_preset.mode,
run_config_mode=run_config.mode),
)
if pipeline_preset.environment_dict is not None:
check.invariant(
(not environment_dict)
or (pipeline_preset.environment_dict == environment_dict),
'The environment set in preset \'{preset}\' does not agree with the environment '
'passed in the `environment_dict` argument.'.format(
preset=preset),
)
environment_dict = pipeline_preset.environment_dict
if pipeline_preset.solid_subset is not None:
pipeline = pipeline.build_sub_pipeline(
pipeline_preset.solid_subset)
check.invariant(
mode is None or mode == pipeline_preset.mode,
'Mode {mode} does not agree with the mode set in preset \'{preset}\': '
'(\'{preset_mode}\')'.format(preset=preset,
preset_mode=pipeline_preset.mode,
mode=mode),
)
mode = pipeline_preset.mode
if run_config.mode is not None or run_config.tags:
warnings.warn((
'In 0.8.0, the use of `run_config` to set pipeline mode and tags will be '
'deprecated. Please use the `mode` and `tags` arguments to `{fn_name}` '
'instead.').format(fn_name=fn_name))
if run_config.mode is not None:
if mode is not None:
check.invariant(
run_config.mode == mode,
'Mode \'{mode}\' does not agree with the mode set in the `run_config`: '
'\'{run_config_mode}\''.format(
mode=mode, run_config_mode=run_config.mode),
)
mode = run_config.mode
if mode is not None:
if not pipeline_def.has_mode_definition(mode):
raise DagsterInvariantViolationError((
'You have attempted to execute pipeline {name} with mode {mode}. '
'Available modes: {modes}').format(
name=pipeline_def.name,
mode=mode,
modes=pipeline_def.available_modes,
))
else:
if not pipeline_def.is_single_mode:
raise DagsterInvariantViolationError((
'Pipeline {name} has multiple modes (Available modes: {modes}) and you have '
'attempted to execute it without specifying a mode. Set '
'mode property on the PipelineRun object.').format(
name=pipeline_def.name,
modes=pipeline_def.available_modes))
mode = pipeline_def.get_default_mode_name()
tags = merge_dicts(merge_dicts(pipeline_def.tags, run_config.tags or {}),
tags)
check.opt_inst_param(instance, 'instance', DagsterInstance)
instance = instance or DagsterInstance.ephemeral()
execution_plan = create_execution_plan(
pipeline,
environment_dict,
mode=mode,
step_keys_to_execute=run_config.step_keys_to_execute,
)
return pipeline, environment_dict, instance, mode, tags, run_config, execution_plan