def create_execution_plan(pipeline, environment_dict=None, mode=None, step_keys_to_execute=None): # backcompat if isinstance(pipeline, PipelineDefinition): pipeline = InMemoryExecutablePipeline(pipeline) check.inst_param(pipeline, 'pipeline', ExecutablePipeline) pipeline_def = pipeline.get_definition() environment_dict = check.opt_dict_param(environment_dict, 'environment_dict', key_type=str) mode = check.opt_str_param(mode, 'mode', default=pipeline_def.get_default_mode_name()) check.opt_list_param(step_keys_to_execute, 'step_keys_to_execute', of_type=str) environment_config = EnvironmentConfig.build(pipeline_def, environment_dict, mode=mode) return ExecutionPlan.build(pipeline, environment_config, mode=mode, step_keys_to_execute=step_keys_to_execute)
def yield_empty_pipeline_context(run_id=None, instance=None): pipeline = InMemoryExecutablePipeline(PipelineDefinition([])) pipeline_def = pipeline.get_definition() instance = check.opt_inst_param( instance, 'instance', DagsterInstance, default=DagsterInstance.ephemeral() ) execution_plan = create_execution_plan(pipeline) pipeline_run = instance.create_run( pipeline_name='<empty>', run_id=run_id, run_config=None, mode=None, solids_to_execute=None, step_keys_to_execute=None, status=None, tags=None, root_run_id=None, parent_run_id=None, pipeline_snapshot=pipeline_def.get_pipeline_snapshot(), execution_plan_snapshot=snapshot_from_execution_plan( execution_plan, pipeline_def.get_pipeline_snapshot_id() ), parent_pipeline_snapshot=pipeline_def.get_parent_pipeline_snapshot(), ) with scoped_pipeline_context(execution_plan, {}, pipeline_run, instance) as context: yield context
def _check_pipeline(pipeline): # backcompat if isinstance(pipeline, PipelineDefinition): pipeline = InMemoryExecutablePipeline(pipeline) check.inst_param(pipeline, 'pipeline', ExecutablePipeline) pipeline_def = pipeline.get_definition() return pipeline, pipeline_def
def _check_execute_pipeline_args(fn_name, pipeline, environment_dict, mode, preset, tags, run_config, instance): # backcompat if isinstance(pipeline, PipelineDefinition): pipeline = InMemoryExecutablePipeline(pipeline) check.inst_param(pipeline, 'pipeline', ExecutablePipeline) pipeline_def = pipeline.get_definition() environment_dict = check.opt_dict_param(environment_dict, 'environment_dict') check.opt_str_param(mode, 'mode') check.opt_str_param(preset, 'preset') check.invariant( not (mode is not None and preset is not None), 'You may set only one of `mode` (got {mode}) or `preset` (got {preset}).' .format(mode=mode, preset=preset), ) tags = check.opt_dict_param(tags, 'tags', key_type=str) run_config = check.opt_inst_param(run_config, 'run_config', RunConfig, default=RunConfig()) if preset is not None: pipeline_preset = pipeline_def.get_preset(preset) check.invariant( run_config.mode is None or pipeline_preset.mode == run_config.mode, 'The mode set in preset \'{preset}\' (\'{preset_mode}\') does not agree with the mode ' 'set in the `run_config` (\'{run_config_mode}\')'.format( preset=preset, preset_mode=pipeline_preset.mode, run_config_mode=run_config.mode), ) if pipeline_preset.environment_dict is not None: check.invariant( (not environment_dict) or (pipeline_preset.environment_dict == environment_dict), 'The environment set in preset \'{preset}\' does not agree with the environment ' 'passed in the `environment_dict` argument.'.format( preset=preset), ) environment_dict = pipeline_preset.environment_dict if pipeline_preset.solid_subset is not None: pipeline = pipeline.build_sub_pipeline( pipeline_preset.solid_subset) check.invariant( mode is None or mode == pipeline_preset.mode, 'Mode {mode} does not agree with the mode set in preset \'{preset}\': ' '(\'{preset_mode}\')'.format(preset=preset, preset_mode=pipeline_preset.mode, mode=mode), ) mode = pipeline_preset.mode if run_config.mode is not None or run_config.tags: warnings.warn(( 'In 0.8.0, the use of `run_config` to set pipeline mode and tags will be ' 'deprecated. Please use the `mode` and `tags` arguments to `{fn_name}` ' 'instead.').format(fn_name=fn_name)) if run_config.mode is not None: if mode is not None: check.invariant( run_config.mode == mode, 'Mode \'{mode}\' does not agree with the mode set in the `run_config`: ' '\'{run_config_mode}\''.format( mode=mode, run_config_mode=run_config.mode), ) mode = run_config.mode if mode is not None: if not pipeline_def.has_mode_definition(mode): raise DagsterInvariantViolationError(( 'You have attempted to execute pipeline {name} with mode {mode}. ' 'Available modes: {modes}').format( name=pipeline_def.name, mode=mode, modes=pipeline_def.available_modes, )) else: if not pipeline_def.is_single_mode: raise DagsterInvariantViolationError(( 'Pipeline {name} has multiple modes (Available modes: {modes}) and you have ' 'attempted to execute it without specifying a mode. Set ' 'mode property on the PipelineRun object.').format( name=pipeline_def.name, modes=pipeline_def.available_modes)) mode = pipeline_def.get_default_mode_name() tags = merge_dicts(merge_dicts(pipeline_def.tags, run_config.tags or {}), tags) check.opt_inst_param(instance, 'instance', DagsterInstance) instance = instance or DagsterInstance.ephemeral() execution_plan = create_execution_plan( pipeline, environment_dict, mode=mode, step_keys_to_execute=run_config.step_keys_to_execute, ) return pipeline, environment_dict, instance, mode, tags, run_config, execution_plan