def create_context_creation_data(
execution_plan, environment_dict, pipeline_run, instance,
):
pipeline_def = execution_plan.pipeline.get_definition()
environment_config = EnvironmentConfig.build(
pipeline_def, environment_dict, mode=pipeline_run.mode
)
mode_def = pipeline_def.get_mode_definition(pipeline_run.mode)
system_storage_def = system_storage_def_from_config(mode_def, environment_config)
executor_def = executor_def_from_config(mode_def, environment_config)
execution_target_handle, _ = ExecutionTargetHandle.get_handle(pipeline_def)
return ContextCreationData(
pipeline=execution_plan.pipeline,
environment_config=environment_config,
pipeline_run=pipeline_run,
mode_def=mode_def,
system_storage_def=system_storage_def,
execution_target_handle=execution_target_handle,
executor_def=executor_def,
instance=instance,
resource_keys_to_init=get_required_resource_keys_to_init(
execution_plan, system_storage_def
),
)
def test_loader_from_default_repository_file_yaml():
handle = handle_for_pipeline_cli_args(
{'pipeline_name': 'foo', 'repository_yaml': script_relative_path('repository_file.yaml')}
)
pipeline = handle.build_pipeline_definition()
assert isinstance(pipeline, PipelineDefinition)
assert pipeline.name == 'foo'
assert ExecutionTargetHandle.get_handle(pipeline) == (handle, None)
def test_load_from_pipeline_module():
handle = handle_for_pipeline_cli_args(
{'module_name': 'dagster_examples.intro_tutorial.repos', 'fn_name': 'hello_cereal_pipeline'}
)
pipeline = handle.build_pipeline_definition()
assert isinstance(pipeline, PipelineDefinition)
assert pipeline.name == 'hello_cereal_pipeline'
assert ExecutionTargetHandle.get_handle(pipeline) == (handle, None)
def test_load_from_pipeline_file():
handle = handle_for_pipeline_cli_args(
{'fn_name': 'define_foo_pipeline', 'python_file': __file__}
)
pipeline = handle.build_pipeline_definition()
assert isinstance(pipeline, PipelineDefinition)
assert pipeline.name == 'foo'
assert ExecutionTargetHandle.get_handle(pipeline) == (handle, None)
def test_load_from_repository_file():
handle = handle_for_pipeline_cli_args(
{'pipeline_name': 'foo', 'python_file': __file__, 'fn_name': 'define_bar_repo'}
)
pipeline = handle.build_pipeline_definition()
assert isinstance(pipeline, PipelineDefinition)
assert pipeline.name == 'foo'
assert ExecutionTargetHandle.get_handle(pipeline) == handle
def test_load_from_repository_module():
handle = handle_for_pipeline_cli_args({
'module_name': 'dagster_examples.intro_tutorial.repos',
'pipeline_name': 'repo_demo_pipeline',
'fn_name': 'define_repo',
})
pipeline = handle.build_pipeline_definition()
assert isinstance(pipeline, PipelineDefinition)
assert pipeline.name == 'repo_demo_pipeline'
assert ExecutionTargetHandle.get_handle(pipeline) == handle
def test_loader_from_default_repository_module_yaml():
handle = handle_for_pipeline_cli_args({
'pipeline_name':
'hello_cereal_pipeline',
'repository_yaml':
file_relative_path(__file__, 'repository_module.yaml'),
})
pipeline = handle.build_pipeline_definition()
assert isinstance(pipeline, PipelineDefinition)
assert pipeline.name == 'hello_cereal_pipeline'
assert ExecutionTargetHandle.get_handle(pipeline) == (handle, None)