def test_repo_mode_data():
assert create_repository_loading_mode_data(
RepositoryTargetInfo(
repository_yaml='foo.yaml',
module_name=None,
python_file=None,
fn_name=None,
)) == RepositoryLoadingModeData(
mode=RepositoryTargetMode.YAML_FILE,
data='foo.yaml',
)
assert create_repository_loading_mode_data(
RepositoryTargetInfo(
repository_yaml=None,
module_name='foo',
python_file=None,
fn_name='define_bar_repo',
)) == RepositoryLoadingModeData(mode=RepositoryTargetMode.MODULE,
data=ModuleTargetFunction(
module_name='foo',
fn_name='define_bar_repo',
))
assert create_repository_loading_mode_data(
RepositoryTargetInfo(
repository_yaml=None,
module_name=None,
python_file='foo.py',
fn_name='define_bar_repo',
)) == RepositoryLoadingModeData(mode=RepositoryTargetMode.FILE,
data=FileTargetFunction(
python_file='foo.py',
fn_name='define_bar_repo',
))
def test_repo_entrypoints():
module = importlib.import_module('dagster.tutorials.intro_tutorial.repos')
assert entrypoint_from_repo_target_info(
RepositoryTargetInfo(
repository_yaml=script_relative_path('repository.yml'),
module_name=None,
python_file=None,
fn_name=None,
)) == LoaderEntrypoint(module,
'dagster.tutorials.intro_tutorial.repos',
'define_repo', {})
module = importlib.import_module('dagster')
assert entrypoint_from_repo_target_info(
RepositoryTargetInfo(repository_yaml=None,
module_name='dagster',
python_file=None,
fn_name='define_bar_repo')) == LoaderEntrypoint(
module, 'dagster', 'define_bar_repo', {})
python_file = script_relative_path('bar_repo.py')
module = imp.load_source('bar_repo', python_file)
assert entrypoint_from_repo_target_info(
RepositoryTargetInfo(
repository_yaml=None,
module_name=None,
python_file=python_file,
fn_name='define_bar_repo',
)) == LoaderEntrypoint(module, 'bar_repo', 'define_bar_repo', {})
def test_execution_crash():
run_id = 'run-1'
repository_container = RepositoryContainer(
RepositoryTargetInfo(
repository_yaml=None,
python_file=__file__,
fn_name='define_crashy_pipeline',
module_name=None,
)
)
pipeline = define_crashy_pipeline()
env_config = {
'solids': {
'sum_solid': {'inputs': {'num': {'csv': {'path': script_relative_path('num.csv')}}}}
}
}
selector = ExecutionSelector('pandas_hello_world')
pipeline_run = InMemoryPipelineRun(
run_id, selector, env_config, create_execution_plan(pipeline, env_config)
)
execution_manager = MultiprocessingExecutionManager()
execution_manager.execute_pipeline(repository_container, pipeline, pipeline_run)
execution_manager.join()
assert pipeline_run.status == PipelineRunStatus.FAILURE
last_log = pipeline_run.all_logs()[-1]
assert last_log.message == (
'Exception: Pipeline execution process for {run_id} unexpectedly exited\n'
).format(run_id=run_id)
def test_failing():
run_id = 'run-1'
repository_container = RepositoryContainer(
RepositoryTargetInfo(
repository_yaml=None,
python_file=__file__,
fn_name='define_failing_pipeline',
module_name=None,
)
)
pipeline = define_failing_pipeline()
env_config = {
'solids': {
'sum_solid': {'inputs': {'num': {'csv': {'path': script_relative_path('num.csv')}}}}
}
}
selector = ExecutionSelector('pandas_hello_world')
pipeline_run = InMemoryPipelineRun(
run_id, selector, env_config, create_execution_plan(pipeline, env_config)
)
execution_manager = MultiprocessingExecutionManager()
execution_manager.execute_pipeline(repository_container, pipeline, pipeline_run)
execution_manager.join()
assert pipeline_run.status == PipelineRunStatus.FAILURE
assert pipeline_run.all_logs()
def test_running():
run_id = 'run-1'
repository_container = RepositoryContainer(
RepositoryTargetInfo(
repository_yaml=None,
python_file=__file__,
fn_name='define_passing_pipeline',
module_name=None,
)
)
pipeline = define_passing_pipeline()
env_config = {
'solids': {
'sum_solid': {'inputs': {'num': {'csv': {'path': script_relative_path('num.csv')}}}}
}
}
selector = ExecutionSelector('pandas_hello_world')
pipeline_run = InMemoryPipelineRun(
run_id, selector, env_config, create_execution_plan(pipeline, env_config)
)
execution_manager = MultiprocessingExecutionManager()
execution_manager.execute_pipeline(repository_container, pipeline, pipeline_run)
execution_manager.join()
assert pipeline_run.status == PipelineRunStatus.SUCCESS
events = pipeline_run.all_logs()
assert events
process_start_events = get_events_of_type(events, EventType.PIPELINE_PROCESS_START)
assert len(process_start_events) == 1
process_started_events = get_events_of_type(events, EventType.PIPELINE_PROCESS_STARTED)
assert len(process_started_events) == 1
def test_repo_file_dynamic_load():
repository = load_repository_object_from_target_info(
RepositoryTargetInfo(
repository_yaml=None,
module_name=None,
python_file=script_relative_path('test_dynamic_loader.py'),
fn_name='define_bar_repo',
)).fn()
assert isinstance(repository, RepositoryDefinition)
assert repository.name == 'bar'
def test_repo_module_dynamic_load():
repository = load_repository_object_from_target_info(
RepositoryTargetInfo(
repository_yaml=None,
module_name='dagster.cli.cli_tests.test_dynamic_loader',
python_file=None,
fn_name='define_bar_repo',
)).fn()
assert isinstance(repository, RepositoryDefinition)
assert repository.name == 'bar'
def test_repo_module_dynamic_load():
repository = load_repository_from_target_info(
RepositoryTargetInfo(
repository_yaml=None,
module_name='dagster.tutorials.intro_tutorial.repos',
python_file=None,
fn_name='define_repo_demo_pipeline',
))
assert isinstance(repository, RepositoryDefinition)
assert repository.name == '<<unnamed>>'
def test_repo_yaml_file_dynamic_load():
repository = load_repository_from_target_info(
RepositoryTargetInfo(
repository_yaml=script_relative_path('repository_file.yml'),
module_name=None,
python_file=None,
fn_name=None,
))
assert isinstance(repository, RepositoryDefinition)
assert repository.name == 'bar'
def test_repo_file_dynamic_load_from_pipeline():
repository = load_repository_object_from_target_info(
RepositoryTargetInfo(
repository_yaml=None,
module_name=None,
python_file=script_relative_path('test_dynamic_loader.py'),
fn_name='define_foo_pipeline',
)).fn()
assert isinstance(repository, RepositoryDefinition)
assert repository.name == '<<unnamed>>'
assert repository.get_pipeline('foo').name == 'foo'
def test_repo_module_dynamic_load_from_pipeline():
repository = load_repository_object_from_target_info(
RepositoryTargetInfo(
repository_yaml=None,
module_name='dagster.cli.cli_tests.test_dynamic_loader',
python_file=None,
fn_name='define_foo_pipeline',
)).fn()
assert isinstance(repository, RepositoryDefinition)
assert repository.name == '<<unnamed>>'
assert repository.get_pipeline('foo').name == 'foo'
