def handle_for_repo_cli_args(kwargs):
'''Builds an ExecutionTargetHandle for CLI arguments, which can be any of the combinations
for repo loading above.
'''
check.dict_param(kwargs, 'kwargs')
_cli_load_invariant(kwargs.get('pipeline_name') is None)
if kwargs.get('repository_yaml') or all_none(kwargs):
_cli_load_invariant(kwargs.get('module_name') is None)
_cli_load_invariant(kwargs.get('python_file') is None)
_cli_load_invariant(kwargs.get('fn_name') is None)
return ExecutionTargetHandle.for_repo_yaml(
repository_yaml=kwargs.get('repository_yaml')
or DEFAULT_REPOSITORY_YAML_FILENAME)
elif kwargs.get('module_name') and kwargs.get('fn_name'):
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('python_file') is None)
return ExecutionTargetHandle.for_repo_module(
module_name=kwargs['module_name'], fn_name=kwargs['fn_name'])
elif kwargs.get('python_file') and kwargs.get('fn_name'):
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('module_name') is None)
return ExecutionTargetHandle.for_repo_python_file(
python_file=kwargs['python_file'], fn_name=kwargs['fn_name'])
else:
raise CliUsageError()
def handle_for_repo_cli_args(kwargs):
'''Builds an ExecutionTargetHandle for CLI arguments, which can be any of the combinations
for repo loading above.
'''
check.dict_param(kwargs, 'kwargs')
_cli_load_invariant(kwargs.get('pipeline_name') is None)
if kwargs.get('repository_yaml') or all_none(kwargs):
_cli_load_invariant(kwargs.get('module_name') is None)
_cli_load_invariant(kwargs.get('python_file') is None)
_cli_load_invariant(kwargs.get('fn_name') is None)
repo_yaml = (os.path.abspath(kwargs.get('repository_yaml'))
if kwargs.get('repository_yaml') else
DEFAULT_REPOSITORY_YAML_FILENAME)
_cli_load_invariant(
os.path.exists(repo_yaml),
'Expected to use file "{}" to load repository but it does not exist. '
'Verify your current working directory or CLI arguments.'.format(
repo_yaml),
)
return ExecutionTargetHandle.for_repo_yaml(repository_yaml=repo_yaml)
elif kwargs.get('module_name') and kwargs.get('fn_name'):
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('python_file') is None)
return ExecutionTargetHandle.for_repo_module(
module_name=kwargs['module_name'], fn_name=kwargs['fn_name'])
elif kwargs.get('python_file') and kwargs.get('fn_name'):
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('module_name') is None)
return ExecutionTargetHandle.for_repo_python_file(
python_file=os.path.abspath(kwargs['python_file']),
fn_name=kwargs['fn_name'])
else:
_cli_load_invariant(False)
def handle_for_pipeline_cli_args(kwargs, use_default_repository_yaml=True):
'''Builds an ExecutionTargetHandle for CLI arguments, which can be any of the combinations
for repo/pipeline loading above.
'''
check.dict_param(kwargs, 'kwargs')
pipeline_name = kwargs.get('pipeline_name')
if pipeline_name and not isinstance(pipeline_name, six.string_types):
if len(pipeline_name) == 1:
pipeline_name = pipeline_name[0]
else:
check.failed(
'Can only handle zero or one pipeline args. Got {pipeline_name}'
.format(pipeline_name=repr(pipeline_name)))
# Pipeline from repository YAML and pipeline_name
if (pipeline_name and kwargs.get('module_name') is None
and kwargs.get('python_file') is None
and (kwargs.get('repository_yaml') is not None
or use_default_repository_yaml)):
_cli_load_invariant(kwargs.get('fn_name') is None)
return ExecutionTargetHandle.for_repo_yaml(
repository_yaml=kwargs.get('repository_yaml') or
DEFAULT_REPOSITORY_YAML_FILENAME).with_pipeline_name(pipeline_name)
# Pipeline from repository python file
elif kwargs.get('python_file') and kwargs.get('fn_name') and pipeline_name:
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('module_name') is None)
return ExecutionTargetHandle.for_repo_python_file(
python_file=os.path.abspath(kwargs['python_file']),
fn_name=kwargs['fn_name']).with_pipeline_name(pipeline_name)
# Pipeline from repository module
elif kwargs.get('module_name') and kwargs.get('fn_name') and pipeline_name:
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('python_file') is None)
return ExecutionTargetHandle.for_repo_module(
module_name=kwargs['module_name'],
fn_name=kwargs['fn_name']).with_pipeline_name(pipeline_name)
# Pipeline from pipeline python file
elif kwargs.get('python_file') and kwargs.get(
'fn_name') and not pipeline_name:
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('module_name') is None)
return ExecutionTargetHandle.for_pipeline_python_file(
python_file=os.path.abspath(kwargs['python_file']),
fn_name=kwargs['fn_name'])
# Pipeline from pipeline module
elif kwargs.get('module_name') and kwargs.get(
'fn_name') and not pipeline_name:
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('python_file') is None)
return ExecutionTargetHandle.for_pipeline_module(
module_name=kwargs['module_name'], fn_name=kwargs['fn_name'])
else:
_cli_load_invariant(False)
def create_context_creation_data(
execution_plan, environment_dict, pipeline_run, instance,
):
pipeline_def = execution_plan.pipeline.get_definition()
environment_config = EnvironmentConfig.build(
pipeline_def, environment_dict, mode=pipeline_run.mode
)
mode_def = pipeline_def.get_mode_definition(pipeline_run.mode)
system_storage_def = system_storage_def_from_config(mode_def, environment_config)
executor_def = executor_def_from_config(mode_def, environment_config)
execution_target_handle, _ = ExecutionTargetHandle.get_handle(pipeline_def)
return ContextCreationData(
pipeline=execution_plan.pipeline,
environment_config=environment_config,
pipeline_run=pipeline_run,
mode_def=mode_def,
system_storage_def=system_storage_def,
execution_target_handle=execution_target_handle,
executor_def=executor_def,
instance=instance,
resource_keys_to_init=get_required_resource_keys_to_init(
execution_plan, system_storage_def
),
)
def test_loader_from_default_repository_file_yaml():
handle = handle_for_pipeline_cli_args(
{'pipeline_name': 'foo', 'repository_yaml': script_relative_path('repository_file.yaml')}
)
pipeline = handle.build_pipeline_definition()
assert isinstance(pipeline, PipelineDefinition)
assert pipeline.name == 'foo'
assert ExecutionTargetHandle.get_handle(pipeline) == (handle, None)
def test_load_from_pipeline_module():
handle = handle_for_pipeline_cli_args(
{'module_name': 'dagster_examples.intro_tutorial.repos', 'fn_name': 'hello_cereal_pipeline'}
)
pipeline = handle.build_pipeline_definition()
assert isinstance(pipeline, PipelineDefinition)
assert pipeline.name == 'hello_cereal_pipeline'
assert ExecutionTargetHandle.get_handle(pipeline) == (handle, None)
def test_load_from_pipeline_file():
handle = handle_for_pipeline_cli_args(
{'fn_name': 'define_foo_pipeline', 'python_file': __file__}
)
pipeline = handle.build_pipeline_definition()
assert isinstance(pipeline, PipelineDefinition)
assert pipeline.name == 'foo'
assert ExecutionTargetHandle.get_handle(pipeline) == (handle, None)
def test_load_from_repository_file():
handle = handle_for_pipeline_cli_args(
{'pipeline_name': 'foo', 'python_file': __file__, 'fn_name': 'define_bar_repo'}
)
pipeline = handle.build_pipeline_definition()
assert isinstance(pipeline, PipelineDefinition)
assert pipeline.name == 'foo'
assert ExecutionTargetHandle.get_handle(pipeline) == handle
def test_load_from_repository_module():
handle = handle_for_pipeline_cli_args({
'module_name': 'dagster_examples.intro_tutorial.repos',
'pipeline_name': 'repo_demo_pipeline',
'fn_name': 'define_repo',
})
pipeline = handle.build_pipeline_definition()
assert isinstance(pipeline, PipelineDefinition)
assert pipeline.name == 'repo_demo_pipeline'
assert ExecutionTargetHandle.get_handle(pipeline) == handle
def test_loader_from_default_repository_module_yaml():
handle = handle_for_pipeline_cli_args({
'pipeline_name':
'hello_cereal_pipeline',
'repository_yaml':
file_relative_path(__file__, 'repository_module.yaml'),
})
pipeline = handle.build_pipeline_definition()
assert isinstance(pipeline, PipelineDefinition)
assert pipeline.name == 'hello_cereal_pipeline'
assert ExecutionTargetHandle.get_handle(pipeline) == (handle, None)