Python create_pipeline_loading_mode_data示例

示例#1

文件： test_dynamic_loader.py 项目： zorrock/dagster

def test_yaml_file():
    module = importlib.import_module('dagster.tutorials.intro_tutorial.repos')

    assert create_pipeline_loading_mode_data(
        PipelineTargetInfo(
            module_name=None,
            pipeline_name='foobar',
            python_file=None,
            fn_name=None,
            repository_yaml=script_relative_path('repository.yml'),
        )) == PipelineLoadingModeData(
            mode=PipelineTargetMode.REPOSITORY,
            data=RepositoryData(
                entrypoint=LoaderEntrypoint(
                    module, 'dagster.tutorials.intro_tutorial.repos',
                    'define_repo', {}),
                pipeline_name='foobar',
            ),
        )

    with pytest.raises(InvalidPipelineLoadingComboError):
        assert create_pipeline_loading_mode_data(
            PipelineTargetInfo(
                module_name='kdjfdk',
                pipeline_name='foobar',
                python_file=None,
                fn_name=None,
                repository_yaml=None,
            ))

    with pytest.raises(InvalidPipelineLoadingComboError):
        assert create_pipeline_loading_mode_data(
            PipelineTargetInfo(
                module_name=None,
                fn_name='kjdfkd',
                pipeline_name='foobar',
                python_file=None,
                repository_yaml=None,
            ))

    with pytest.raises(InvalidPipelineLoadingComboError):
        assert create_pipeline_loading_mode_data(
            PipelineTargetInfo(
                module_name=None,
                pipeline_name='foobar',
                python_file='kjdfkdj',
                fn_name=None,
                repository_yaml=None,
            ))

示例#2

def test_yaml_file():
    assert create_pipeline_loading_mode_data(
        PipelineTargetInfo(
            module_name=None,
            pipeline_name='foobar',
            python_file=None,
            fn_name=None,
            repository_yaml='some_file.yml',
        )) == PipelineLoadingModeData(
            mode=PipelineTargetMode.REPOSITORY_YAML_FILE,
            data=RepositoryYamlData(repository_yaml='some_file.yml',
                                    pipeline_name='foobar'),
        )

    with pytest.raises(InvalidPipelineLoadingComboError):
        assert create_pipeline_loading_mode_data(
            PipelineTargetInfo(
                module_name='kdjfdk',
                pipeline_name='foobar',
                python_file=None,
                fn_name=None,
                repository_yaml=None,
            ))

    with pytest.raises(InvalidPipelineLoadingComboError):
        assert create_pipeline_loading_mode_data(
            PipelineTargetInfo(
                module_name=None,
                fn_name='kjdfkd',
                pipeline_name='foobar',
                python_file=None,
                repository_yaml=None,
            ))

    with pytest.raises(InvalidPipelineLoadingComboError):
        assert create_pipeline_loading_mode_data(
            PipelineTargetInfo(
                module_name=None,
                pipeline_name='foobar',
                python_file='kjdfkdj',
                fn_name=None,
                repository_yaml=None,
            ))

示例#3

文件： test_dynamic_loader.py 项目： zorrock/dagster

def test_repository_python_file():
    python_file = script_relative_path('bar_repo.py')
    module = imp.load_source('bar_repo', python_file)

    mode_data = create_pipeline_loading_mode_data(
        PipelineTargetInfo(
            module_name=None,
            pipeline_name='foo',
            python_file=python_file,
            fn_name='define_bar_repo',
            repository_yaml=None,
        ))

    assert mode_data == PipelineLoadingModeData(
        mode=PipelineTargetMode.REPOSITORY,
        data=RepositoryData(
            entrypoint=LoaderEntrypoint(module, 'bar_repo', 'define_bar_repo',
                                        {}),
            pipeline_name='foo',
        ),
    )

    with pytest.raises(InvalidPipelineLoadingComboError):
        create_pipeline_loading_mode_data(
            PipelineTargetInfo(
                module_name='kdjfkd',
                pipeline_name='foo',
                python_file=script_relative_path('bar_repo.py'),
                fn_name='define_bar_repo',
                repository_yaml=None,
            ))

    with pytest.raises(InvalidPipelineLoadingComboError):
        create_pipeline_loading_mode_data(
            PipelineTargetInfo(
                module_name=None,
                pipeline_name='foo',
                python_file=script_relative_path('bar_repo.py'),
                fn_name='define_bar_repo',
                repository_yaml='kjdfkdjf',
            ))

示例#4

def test_repository_python_file():
    mode_data = create_pipeline_loading_mode_data(
        PipelineTargetInfo(
            module_name=None,
            pipeline_name='foo',
            python_file='bar_repo.py',
            fn_name='define_bar_repo',
            repository_yaml=None,
        ))

    assert mode_data == PipelineLoadingModeData(
        mode=PipelineTargetMode.REPOSITORY_PYTHON_FILE,
        data=RepositoryPythonFileData(
            file_target_function=FileTargetFunction(
                python_file='bar_repo.py',
                fn_name='define_bar_repo',
            ),
            pipeline_name='foo',
        ))

    with pytest.raises(InvalidPipelineLoadingComboError):
        create_pipeline_loading_mode_data(
            PipelineTargetInfo(
                module_name='kdjfkd',
                pipeline_name='foo',
                python_file='bar_repo.py',
                fn_name='define_bar_repo',
                repository_yaml=None,
            ))

    with pytest.raises(InvalidPipelineLoadingComboError):
        create_pipeline_loading_mode_data(
            PipelineTargetInfo(
                module_name=None,
                pipeline_name='foo',
                python_file='bar_repo.py',
                fn_name='define_bar_repo',
                repository_yaml='kjdfkdjf',
            ))

示例#5

文件： test_dynamic_loader.py 项目： zorrock/dagster

def test_pipeline_module():
    mode_data = create_pipeline_loading_mode_data(
        PipelineTargetInfo(
            module_name='dagster',
            fn_name='define_pipeline',
            pipeline_name=None,
            python_file=None,
            repository_yaml=None,
        ))

    assert mode_data == PipelineLoadingModeData(
        mode=PipelineTargetMode.PIPELINE,
        data=LoaderEntrypoint(importlib.import_module('dagster'), 'dagster',
                              'define_pipeline', {}),
    )

示例#6

def test_pipeline_python_file():
    mode_data = create_pipeline_loading_mode_data(
        PipelineTargetInfo(
            module_name=None,
            fn_name='define_pipeline',
            pipeline_name=None,
            python_file='foo.py',
            repository_yaml=None,
        ))

    assert mode_data == PipelineLoadingModeData(
        mode=PipelineTargetMode.PIPELINE_PYTHON_FILE,
        data=FileTargetFunction(python_file='foo.py',
                                fn_name='define_pipeline'),
    )

示例#7

文件： test_dynamic_loader.py 项目： zorrock/dagster

def test_pipeline_python_file():
    python_file = script_relative_path('foo_pipeline.py')
    module = imp.load_source('foo_pipeline', python_file)

    mode_data = create_pipeline_loading_mode_data(
        PipelineTargetInfo(
            module_name=None,
            fn_name='define_pipeline',
            pipeline_name=None,
            python_file=python_file,
            repository_yaml=None,
        ))

    assert mode_data == PipelineLoadingModeData(
        mode=PipelineTargetMode.PIPELINE,
        data=LoaderEntrypoint(module, 'foo_pipeline', 'define_pipeline', {}),
    )

示例#8

def test_repository_module():
    mode_data = create_pipeline_loading_mode_data(
        PipelineTargetInfo(
            module_name='bar_module',
            pipeline_name='foo',
            python_file=None,
            fn_name='define_bar_repo',
            repository_yaml=None,
        ))

    assert mode_data == PipelineLoadingModeData(
        mode=PipelineTargetMode.REPOSITORY_MODULE,
        data=RepositoryModuleData(
            module_target_function=ModuleTargetFunction(
                module_name='bar_module', fn_name='define_bar_repo'),
            pipeline_name='foo',
        ),
    )

示例#9

文件： test_dynamic_loader.py 项目： zorrock/dagster

def test_repository_module():
    module = importlib.import_module('dagster')
    mode_data = create_pipeline_loading_mode_data(
        PipelineTargetInfo(
            module_name='dagster',
            pipeline_name='foo',
            python_file=None,
            fn_name='define_bar_repo',
            repository_yaml=None,
        ))

    assert mode_data == PipelineLoadingModeData(
        mode=PipelineTargetMode.REPOSITORY,
        data=RepositoryData(
            entrypoint=LoaderEntrypoint(module, 'dagster', 'define_bar_repo',
                                        {}),
            pipeline_name='foo',
        ),
    )