def handle_for_repo_cli_args(kwargs): '''Builds an ExecutionTargetHandle for CLI arguments, which can be any of the combinations for repo loading above. ''' check.dict_param(kwargs, 'kwargs') _cli_load_invariant(kwargs.get('pipeline_name') is None) if kwargs.get('repository_yaml') or all_none(kwargs): _cli_load_invariant(kwargs.get('module_name') is None) _cli_load_invariant(kwargs.get('python_file') is None) _cli_load_invariant(kwargs.get('fn_name') is None) return ExecutionTargetHandle.for_repo_yaml( repository_yaml=kwargs.get('repository_yaml') or DEFAULT_REPOSITORY_YAML_FILENAME) elif kwargs.get('module_name') and kwargs.get('fn_name'): _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('python_file') is None) return ExecutionTargetHandle.for_repo_module( module_name=kwargs['module_name'], fn_name=kwargs['fn_name']) elif kwargs.get('python_file') and kwargs.get('fn_name'): _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('module_name') is None) return ExecutionTargetHandle.for_repo_python_file( python_file=kwargs['python_file'], fn_name=kwargs['fn_name']) else: raise CliUsageError()
def handle_for_repo_cli_args(kwargs): '''Builds an ExecutionTargetHandle for CLI arguments, which can be any of the combinations for repo loading above. ''' check.dict_param(kwargs, 'kwargs') _cli_load_invariant(kwargs.get('pipeline_name') is None) if kwargs.get('repository_yaml') or all_none(kwargs): _cli_load_invariant(kwargs.get('module_name') is None) _cli_load_invariant(kwargs.get('python_file') is None) _cli_load_invariant(kwargs.get('fn_name') is None) repo_yaml = (os.path.abspath(kwargs.get('repository_yaml')) if kwargs.get('repository_yaml') else DEFAULT_REPOSITORY_YAML_FILENAME) _cli_load_invariant( os.path.exists(repo_yaml), 'Expected to use file "{}" to load repository but it does not exist. ' 'Verify your current working directory or CLI arguments.'.format( repo_yaml), ) return ExecutionTargetHandle.for_repo_yaml(repository_yaml=repo_yaml) elif kwargs.get('module_name') and kwargs.get('fn_name'): _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('python_file') is None) return ExecutionTargetHandle.for_repo_module( module_name=kwargs['module_name'], fn_name=kwargs['fn_name']) elif kwargs.get('python_file') and kwargs.get('fn_name'): _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('module_name') is None) return ExecutionTargetHandle.for_repo_python_file( python_file=os.path.abspath(kwargs['python_file']), fn_name=kwargs['fn_name']) else: _cli_load_invariant(False)
def handle_for_pipeline_cli_args(kwargs, use_default_repository_yaml=True): '''Builds an ExecutionTargetHandle for CLI arguments, which can be any of the combinations for repo/pipeline loading above. ''' check.dict_param(kwargs, 'kwargs') pipeline_name = kwargs.get('pipeline_name') if pipeline_name and not isinstance(pipeline_name, six.string_types): if len(pipeline_name) == 1: pipeline_name = pipeline_name[0] else: check.failed( 'Can only handle zero or one pipeline args. Got {pipeline_name}' .format(pipeline_name=repr(pipeline_name))) # Pipeline from repository YAML and pipeline_name if (pipeline_name and kwargs.get('module_name') is None and kwargs.get('python_file') is None and (kwargs.get('repository_yaml') is not None or use_default_repository_yaml)): _cli_load_invariant(kwargs.get('fn_name') is None) return ExecutionTargetHandle.for_repo_yaml( repository_yaml=kwargs.get('repository_yaml') or DEFAULT_REPOSITORY_YAML_FILENAME).with_pipeline_name(pipeline_name) # Pipeline from repository python file elif kwargs.get('python_file') and kwargs.get('fn_name') and pipeline_name: _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('module_name') is None) return ExecutionTargetHandle.for_repo_python_file( python_file=os.path.abspath(kwargs['python_file']), fn_name=kwargs['fn_name']).with_pipeline_name(pipeline_name) # Pipeline from repository module elif kwargs.get('module_name') and kwargs.get('fn_name') and pipeline_name: _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('python_file') is None) return ExecutionTargetHandle.for_repo_module( module_name=kwargs['module_name'], fn_name=kwargs['fn_name']).with_pipeline_name(pipeline_name) # Pipeline from pipeline python file elif kwargs.get('python_file') and kwargs.get( 'fn_name') and not pipeline_name: _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('module_name') is None) return ExecutionTargetHandle.for_pipeline_python_file( python_file=os.path.abspath(kwargs['python_file']), fn_name=kwargs['fn_name']) # Pipeline from pipeline module elif kwargs.get('module_name') and kwargs.get( 'fn_name') and not pipeline_name: _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('python_file') is None) return ExecutionTargetHandle.for_pipeline_module( module_name=kwargs['module_name'], fn_name=kwargs['fn_name']) else: _cli_load_invariant(False)
def create_context_creation_data( execution_plan, environment_dict, pipeline_run, instance, ): pipeline_def = execution_plan.pipeline.get_definition() environment_config = EnvironmentConfig.build( pipeline_def, environment_dict, mode=pipeline_run.mode ) mode_def = pipeline_def.get_mode_definition(pipeline_run.mode) system_storage_def = system_storage_def_from_config(mode_def, environment_config) executor_def = executor_def_from_config(mode_def, environment_config) execution_target_handle, _ = ExecutionTargetHandle.get_handle(pipeline_def) return ContextCreationData( pipeline=execution_plan.pipeline, environment_config=environment_config, pipeline_run=pipeline_run, mode_def=mode_def, system_storage_def=system_storage_def, execution_target_handle=execution_target_handle, executor_def=executor_def, instance=instance, resource_keys_to_init=get_required_resource_keys_to_init( execution_plan, system_storage_def ), )
def test_loader_from_default_repository_file_yaml(): handle = handle_for_pipeline_cli_args( {'pipeline_name': 'foo', 'repository_yaml': script_relative_path('repository_file.yaml')} ) pipeline = handle.build_pipeline_definition() assert isinstance(pipeline, PipelineDefinition) assert pipeline.name == 'foo' assert ExecutionTargetHandle.get_handle(pipeline) == (handle, None)
def test_load_from_pipeline_module(): handle = handle_for_pipeline_cli_args( {'module_name': 'dagster_examples.intro_tutorial.repos', 'fn_name': 'hello_cereal_pipeline'} ) pipeline = handle.build_pipeline_definition() assert isinstance(pipeline, PipelineDefinition) assert pipeline.name == 'hello_cereal_pipeline' assert ExecutionTargetHandle.get_handle(pipeline) == (handle, None)
def test_load_from_pipeline_file(): handle = handle_for_pipeline_cli_args( {'fn_name': 'define_foo_pipeline', 'python_file': __file__} ) pipeline = handle.build_pipeline_definition() assert isinstance(pipeline, PipelineDefinition) assert pipeline.name == 'foo' assert ExecutionTargetHandle.get_handle(pipeline) == (handle, None)
def test_load_from_repository_file(): handle = handle_for_pipeline_cli_args( {'pipeline_name': 'foo', 'python_file': __file__, 'fn_name': 'define_bar_repo'} ) pipeline = handle.build_pipeline_definition() assert isinstance(pipeline, PipelineDefinition) assert pipeline.name == 'foo' assert ExecutionTargetHandle.get_handle(pipeline) == handle
def test_load_from_repository_module(): handle = handle_for_pipeline_cli_args({ 'module_name': 'dagster_examples.intro_tutorial.repos', 'pipeline_name': 'repo_demo_pipeline', 'fn_name': 'define_repo', }) pipeline = handle.build_pipeline_definition() assert isinstance(pipeline, PipelineDefinition) assert pipeline.name == 'repo_demo_pipeline' assert ExecutionTargetHandle.get_handle(pipeline) == handle
def test_loader_from_default_repository_module_yaml(): handle = handle_for_pipeline_cli_args({ 'pipeline_name': 'hello_cereal_pipeline', 'repository_yaml': file_relative_path(__file__, 'repository_module.yaml'), }) pipeline = handle.build_pipeline_definition() assert isinstance(pipeline, PipelineDefinition) assert pipeline.name == 'hello_cereal_pipeline' assert ExecutionTargetHandle.get_handle(pipeline) == (handle, None)