def test_yaml_file():
module = importlib.import_module('dagster.tutorials.intro_tutorial.repos')
assert create_pipeline_loading_mode_data(
PipelineTargetInfo(
module_name=None,
pipeline_name='foobar',
python_file=None,
fn_name=None,
repository_yaml=script_relative_path('repository.yml'),
)) == PipelineLoadingModeData(
mode=PipelineTargetMode.REPOSITORY,
data=RepositoryData(
entrypoint=LoaderEntrypoint(
module, 'dagster.tutorials.intro_tutorial.repos',
'define_repo', {}),
pipeline_name='foobar',
),
)
with pytest.raises(InvalidPipelineLoadingComboError):
assert create_pipeline_loading_mode_data(
PipelineTargetInfo(
module_name='kdjfdk',
pipeline_name='foobar',
python_file=None,
fn_name=None,
repository_yaml=None,
))
with pytest.raises(InvalidPipelineLoadingComboError):
assert create_pipeline_loading_mode_data(
PipelineTargetInfo(
module_name=None,
fn_name='kjdfkd',
pipeline_name='foobar',
python_file=None,
repository_yaml=None,
))
with pytest.raises(InvalidPipelineLoadingComboError):
assert create_pipeline_loading_mode_data(
PipelineTargetInfo(
module_name=None,
pipeline_name='foobar',
python_file='kjdfkdj',
fn_name=None,
repository_yaml=None,
))
def test_yaml_file():
assert create_pipeline_loading_mode_data(
PipelineTargetInfo(
module_name=None,
pipeline_name='foobar',
python_file=None,
fn_name=None,
repository_yaml='some_file.yml',
)) == PipelineLoadingModeData(
mode=PipelineTargetMode.REPOSITORY_YAML_FILE,
data=RepositoryYamlData(repository_yaml='some_file.yml',
pipeline_name='foobar'),
)
with pytest.raises(InvalidPipelineLoadingComboError):
assert create_pipeline_loading_mode_data(
PipelineTargetInfo(
module_name='kdjfdk',
pipeline_name='foobar',
python_file=None,
fn_name=None,
repository_yaml=None,
))
with pytest.raises(InvalidPipelineLoadingComboError):
assert create_pipeline_loading_mode_data(
PipelineTargetInfo(
module_name=None,
fn_name='kjdfkd',
pipeline_name='foobar',
python_file=None,
repository_yaml=None,
))
with pytest.raises(InvalidPipelineLoadingComboError):
assert create_pipeline_loading_mode_data(
PipelineTargetInfo(
module_name=None,
pipeline_name='foobar',
python_file='kjdfkdj',
fn_name=None,
repository_yaml=None,
))
def test_repository_python_file():
python_file = script_relative_path('bar_repo.py')
module = imp.load_source('bar_repo', python_file)
mode_data = create_pipeline_loading_mode_data(
PipelineTargetInfo(
module_name=None,
pipeline_name='foo',
python_file=python_file,
fn_name='define_bar_repo',
repository_yaml=None,
))
assert mode_data == PipelineLoadingModeData(
mode=PipelineTargetMode.REPOSITORY,
data=RepositoryData(
entrypoint=LoaderEntrypoint(module, 'bar_repo', 'define_bar_repo',
{}),
pipeline_name='foo',
),
)
with pytest.raises(InvalidPipelineLoadingComboError):
create_pipeline_loading_mode_data(
PipelineTargetInfo(
module_name='kdjfkd',
pipeline_name='foo',
python_file=script_relative_path('bar_repo.py'),
fn_name='define_bar_repo',
repository_yaml=None,
))
with pytest.raises(InvalidPipelineLoadingComboError):
create_pipeline_loading_mode_data(
PipelineTargetInfo(
module_name=None,
pipeline_name='foo',
python_file=script_relative_path('bar_repo.py'),
fn_name='define_bar_repo',
repository_yaml='kjdfkdjf',
))
def test_loader_from_default_repository_file_yaml():
pipeline = load_pipeline_from_target_info(
PipelineTargetInfo(
module_name=None,
pipeline_name='foo',
python_file=None,
fn_name=None,
repository_yaml=script_relative_path('repository_file.yml'),
))
assert isinstance(pipeline, PipelineDefinition)
assert pipeline.name == 'foo'
def test_load_from_pipeline_module():
pipeline = load_pipeline_from_target_info(
PipelineTargetInfo(
module_name='dagster.cli.cli_tests.test_dynamic_loader',
fn_name='define_foo_pipeline',
pipeline_name=None,
python_file=None,
repository_yaml=None,
))
assert isinstance(pipeline, PipelineDefinition)
assert pipeline.name == 'foo'
def test_load_from_pipeline_file():
pipeline = load_pipeline_from_target_info(
PipelineTargetInfo(
module_name=None,
fn_name='define_foo_pipeline',
pipeline_name=None,
python_file=script_relative_path('test_dynamic_loader.py'),
repository_yaml=None,
))
assert isinstance(pipeline, PipelineDefinition)
assert pipeline.name == 'foo'
def test_load_from_pipeline_module():
pipeline = load_pipeline_from_target_info(
PipelineTargetInfo(
module_name='dagster.tutorials.intro_tutorial.repos',
fn_name='define_repo_demo_pipeline',
pipeline_name=None,
python_file=None,
repository_yaml=None,
))
assert isinstance(pipeline, PipelineDefinition)
assert pipeline.name == 'repo_demo_pipeline'
def test_repository_python_file():
mode_data = create_pipeline_loading_mode_data(
PipelineTargetInfo(
module_name=None,
pipeline_name='foo',
python_file='bar_repo.py',
fn_name='define_bar_repo',
repository_yaml=None,
))
assert mode_data == PipelineLoadingModeData(
mode=PipelineTargetMode.REPOSITORY_PYTHON_FILE,
data=RepositoryPythonFileData(
file_target_function=FileTargetFunction(
python_file='bar_repo.py',
fn_name='define_bar_repo',
),
pipeline_name='foo',
))
with pytest.raises(InvalidPipelineLoadingComboError):
create_pipeline_loading_mode_data(
PipelineTargetInfo(
module_name='kdjfkd',
pipeline_name='foo',
python_file='bar_repo.py',
fn_name='define_bar_repo',
repository_yaml=None,
))
with pytest.raises(InvalidPipelineLoadingComboError):
create_pipeline_loading_mode_data(
PipelineTargetInfo(
module_name=None,
pipeline_name='foo',
python_file='bar_repo.py',
fn_name='define_bar_repo',
repository_yaml='kjdfkdjf',
))
def test_pipeline_module():
mode_data = create_pipeline_loading_mode_data(
PipelineTargetInfo(
module_name='dagster',
fn_name='define_pipeline',
pipeline_name=None,
python_file=None,
repository_yaml=None,
))
assert mode_data == PipelineLoadingModeData(
mode=PipelineTargetMode.PIPELINE,
data=LoaderEntrypoint(importlib.import_module('dagster'), 'dagster',
'define_pipeline', {}),
)
def test_pipeline_python_file():
mode_data = create_pipeline_loading_mode_data(
PipelineTargetInfo(
module_name=None,
fn_name='define_pipeline',
pipeline_name=None,
python_file='foo.py',
repository_yaml=None,
))
assert mode_data == PipelineLoadingModeData(
mode=PipelineTargetMode.PIPELINE_PYTHON_FILE,
data=FileTargetFunction(python_file='foo.py',
fn_name='define_pipeline'),
)
def test_pipeline_python_file():
python_file = script_relative_path('foo_pipeline.py')
module = imp.load_source('foo_pipeline', python_file)
mode_data = create_pipeline_loading_mode_data(
PipelineTargetInfo(
module_name=None,
fn_name='define_pipeline',
pipeline_name=None,
python_file=python_file,
repository_yaml=None,
))
assert mode_data == PipelineLoadingModeData(
mode=PipelineTargetMode.PIPELINE,
data=LoaderEntrypoint(module, 'foo_pipeline', 'define_pipeline', {}),
)
def test_repository_module():
mode_data = create_pipeline_loading_mode_data(
PipelineTargetInfo(
module_name='bar_module',
pipeline_name='foo',
python_file=None,
fn_name='define_bar_repo',
repository_yaml=None,
))
assert mode_data == PipelineLoadingModeData(
mode=PipelineTargetMode.REPOSITORY_MODULE,
data=RepositoryModuleData(
module_target_function=ModuleTargetFunction(
module_name='bar_module', fn_name='define_bar_repo'),
pipeline_name='foo',
),
)
def test_repository_module():
module = importlib.import_module('dagster')
mode_data = create_pipeline_loading_mode_data(
PipelineTargetInfo(
module_name='dagster',
pipeline_name='foo',
python_file=None,
fn_name='define_bar_repo',
repository_yaml=None,
))
assert mode_data == PipelineLoadingModeData(
mode=PipelineTargetMode.REPOSITORY,
data=RepositoryData(
entrypoint=LoaderEntrypoint(module, 'dagster', 'define_bar_repo',
{}),
pipeline_name='foo',
),
)