def test_repo_mode_data(): assert create_repository_loading_mode_data( RepositoryTargetInfo( repository_yaml='foo.yaml', module_name=None, python_file=None, fn_name=None, )) == RepositoryLoadingModeData( mode=RepositoryTargetMode.YAML_FILE, data='foo.yaml', ) assert create_repository_loading_mode_data( RepositoryTargetInfo( repository_yaml=None, module_name='foo', python_file=None, fn_name='define_bar_repo', )) == RepositoryLoadingModeData(mode=RepositoryTargetMode.MODULE, data=ModuleTargetFunction( module_name='foo', fn_name='define_bar_repo', )) assert create_repository_loading_mode_data( RepositoryTargetInfo( repository_yaml=None, module_name=None, python_file='foo.py', fn_name='define_bar_repo', )) == RepositoryLoadingModeData(mode=RepositoryTargetMode.FILE, data=FileTargetFunction( python_file='foo.py', fn_name='define_bar_repo', ))
def test_repo_entrypoints(): module = importlib.import_module('dagster.tutorials.intro_tutorial.repos') assert entrypoint_from_repo_target_info( RepositoryTargetInfo( repository_yaml=script_relative_path('repository.yml'), module_name=None, python_file=None, fn_name=None, )) == LoaderEntrypoint(module, 'dagster.tutorials.intro_tutorial.repos', 'define_repo', {}) module = importlib.import_module('dagster') assert entrypoint_from_repo_target_info( RepositoryTargetInfo(repository_yaml=None, module_name='dagster', python_file=None, fn_name='define_bar_repo')) == LoaderEntrypoint( module, 'dagster', 'define_bar_repo', {}) python_file = script_relative_path('bar_repo.py') module = imp.load_source('bar_repo', python_file) assert entrypoint_from_repo_target_info( RepositoryTargetInfo( repository_yaml=None, module_name=None, python_file=python_file, fn_name='define_bar_repo', )) == LoaderEntrypoint(module, 'bar_repo', 'define_bar_repo', {})
def test_execution_crash(): run_id = 'run-1' repository_container = RepositoryContainer( RepositoryTargetInfo( repository_yaml=None, python_file=__file__, fn_name='define_crashy_pipeline', module_name=None, ) ) pipeline = define_crashy_pipeline() env_config = { 'solids': { 'sum_solid': {'inputs': {'num': {'csv': {'path': script_relative_path('num.csv')}}}} } } selector = ExecutionSelector('pandas_hello_world') pipeline_run = InMemoryPipelineRun( run_id, selector, env_config, create_execution_plan(pipeline, env_config) ) execution_manager = MultiprocessingExecutionManager() execution_manager.execute_pipeline(repository_container, pipeline, pipeline_run) execution_manager.join() assert pipeline_run.status == PipelineRunStatus.FAILURE last_log = pipeline_run.all_logs()[-1] assert last_log.message == ( 'Exception: Pipeline execution process for {run_id} unexpectedly exited\n' ).format(run_id=run_id)
def test_failing(): run_id = 'run-1' repository_container = RepositoryContainer( RepositoryTargetInfo( repository_yaml=None, python_file=__file__, fn_name='define_failing_pipeline', module_name=None, ) ) pipeline = define_failing_pipeline() env_config = { 'solids': { 'sum_solid': {'inputs': {'num': {'csv': {'path': script_relative_path('num.csv')}}}} } } selector = ExecutionSelector('pandas_hello_world') pipeline_run = InMemoryPipelineRun( run_id, selector, env_config, create_execution_plan(pipeline, env_config) ) execution_manager = MultiprocessingExecutionManager() execution_manager.execute_pipeline(repository_container, pipeline, pipeline_run) execution_manager.join() assert pipeline_run.status == PipelineRunStatus.FAILURE assert pipeline_run.all_logs()
def test_running(): run_id = 'run-1' repository_container = RepositoryContainer( RepositoryTargetInfo( repository_yaml=None, python_file=__file__, fn_name='define_passing_pipeline', module_name=None, ) ) pipeline = define_passing_pipeline() env_config = { 'solids': { 'sum_solid': {'inputs': {'num': {'csv': {'path': script_relative_path('num.csv')}}}} } } selector = ExecutionSelector('pandas_hello_world') pipeline_run = InMemoryPipelineRun( run_id, selector, env_config, create_execution_plan(pipeline, env_config) ) execution_manager = MultiprocessingExecutionManager() execution_manager.execute_pipeline(repository_container, pipeline, pipeline_run) execution_manager.join() assert pipeline_run.status == PipelineRunStatus.SUCCESS events = pipeline_run.all_logs() assert events process_start_events = get_events_of_type(events, EventType.PIPELINE_PROCESS_START) assert len(process_start_events) == 1 process_started_events = get_events_of_type(events, EventType.PIPELINE_PROCESS_STARTED) assert len(process_started_events) == 1
def test_repo_file_dynamic_load(): repository = load_repository_object_from_target_info( RepositoryTargetInfo( repository_yaml=None, module_name=None, python_file=script_relative_path('test_dynamic_loader.py'), fn_name='define_bar_repo', )).fn() assert isinstance(repository, RepositoryDefinition) assert repository.name == 'bar'
def test_repo_module_dynamic_load(): repository = load_repository_object_from_target_info( RepositoryTargetInfo( repository_yaml=None, module_name='dagster.cli.cli_tests.test_dynamic_loader', python_file=None, fn_name='define_bar_repo', )).fn() assert isinstance(repository, RepositoryDefinition) assert repository.name == 'bar'
def test_repo_module_dynamic_load(): repository = load_repository_from_target_info( RepositoryTargetInfo( repository_yaml=None, module_name='dagster.tutorials.intro_tutorial.repos', python_file=None, fn_name='define_repo_demo_pipeline', )) assert isinstance(repository, RepositoryDefinition) assert repository.name == '<<unnamed>>'
def test_repo_yaml_file_dynamic_load(): repository = load_repository_from_target_info( RepositoryTargetInfo( repository_yaml=script_relative_path('repository_file.yml'), module_name=None, python_file=None, fn_name=None, )) assert isinstance(repository, RepositoryDefinition) assert repository.name == 'bar'
def test_repo_file_dynamic_load_from_pipeline(): repository = load_repository_object_from_target_info( RepositoryTargetInfo( repository_yaml=None, module_name=None, python_file=script_relative_path('test_dynamic_loader.py'), fn_name='define_foo_pipeline', )).fn() assert isinstance(repository, RepositoryDefinition) assert repository.name == '<<unnamed>>' assert repository.get_pipeline('foo').name == 'foo'
def test_repo_module_dynamic_load_from_pipeline(): repository = load_repository_object_from_target_info( RepositoryTargetInfo( repository_yaml=None, module_name='dagster.cli.cli_tests.test_dynamic_loader', python_file=None, fn_name='define_foo_pipeline', )).fn() assert isinstance(repository, RepositoryDefinition) assert repository.name == '<<unnamed>>' assert repository.get_pipeline('foo').name == 'foo'