def test_create_app_with_external_repo_data(): recon_repo = ReconstructableRepository.from_yaml( file_relative_path(__file__, './repository.yaml')) assert create_app_with_external_repository( ExternalRepository.from_repository_def(recon_repo.get_definition()), DagsterInstance.ephemeral(), )
def define_context_for_repository_yaml(path, instance): check.inst_param(instance, 'instance', DagsterInstance) return DagsterGraphQLInProcessRepositoryContext( recon_repo=ReconstructableRepository.from_yaml(path), instance=instance, execution_manager=SynchronousExecutionManager(), )
def recon_repo_for_cli_args(kwargs): '''Builds a ReconstructableRepository for CLI arguments, which can be any of the combinations for repo loading above. ''' check.dict_param(kwargs, 'kwargs') _cli_load_invariant(kwargs.get('pipeline_name') is None) if kwargs.get('repository_yaml') or all_none(kwargs): _cli_load_invariant(kwargs.get('module_name') is None) _cli_load_invariant(kwargs.get('python_file') is None) _cli_load_invariant(kwargs.get('fn_name') is None) repo_yaml = ( os.path.abspath(kwargs.get('repository_yaml')) if kwargs.get('repository_yaml') else DEFAULT_REPOSITORY_YAML_FILENAME ) _cli_load_invariant( os.path.exists(repo_yaml), 'Expected to use file "{}" to load repository but it does not exist. ' 'Verify your current working directory or CLI arguments.'.format(repo_yaml), ) return ReconstructableRepository.from_yaml(repo_yaml) elif kwargs.get('module_name') and kwargs.get('fn_name'): _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('python_file') is None) return ReconstructableRepository.for_module(kwargs['module_name'], kwargs['fn_name']) elif kwargs.get('python_file') and kwargs.get('fn_name'): _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('module_name') is None) return ReconstructableRepository.for_file( os.path.abspath(kwargs['python_file']), kwargs['fn_name'] ) else: _cli_load_invariant(False)
def test_successful_host_dagit_ui(): with mock.patch('gevent.pywsgi.WSGIServer'), seven.TemporaryDirectory() as temp_dir: recon_repo = ReconstructableRepository.from_yaml( file_relative_path(__file__, './repository.yaml') ) host_dagit_ui_with_reconstructable_repo( storage_fallback=temp_dir, recon_repo=recon_repo, host=None, port=2343 )
def test_index_view(): with create_app_with_reconstructable_repo( ReconstructableRepository.from_yaml(file_relative_path(__file__, './repository.yaml')), DagsterInstance.ephemeral(), ).test_client() as client: res = client.get('/') assert res.status_code == 200, res.data assert b'You need to enable JavaScript to run this app' in res.data
def get_full_external_pipeline(repo_yaml, pipeline_name): recon_repo = ReconstructableRepository.from_yaml(repo_yaml) repo_def = recon_repo.get_definition() external_repo = external_repo_from_def( repo_def, RepositoryHandle( repo_def.name, LocationHandle('<<MOCK_FOR_TEST>>', recon_repo.pointer), ), ) return external_repo.get_full_external_pipeline(pipeline_name)
def define_context_for_repository_yaml(path, instance): check.inst_param(instance, 'instance', DagsterInstance) return DagsterGraphQLContext( environments=[ InProcessDagsterEnvironment( ReconstructableRepository.from_yaml(path), execution_manager=SynchronousExecutionManager(), ) ], instance=instance, )
def test_notebook_view(): notebook_path = file_relative_path(__file__, 'render_uuid_notebook.ipynb') with create_app_with_reconstructable_repo( ReconstructableRepository.from_yaml(file_relative_path(__file__, './repository.yaml')), DagsterInstance.ephemeral(), ).test_client() as client: res = client.get('/dagit/notebook?path={}'.format(notebook_path)) assert res.status_code == 200 # This magic guid is hardcoded in the notebook assert b'6cac0c38-2c97-49ca-887c-4ac43f141213' in res.data
def test_unknown_error(): class AnException(Exception): pass def _raise_custom_error(): raise AnException('foobar') with mock.patch( 'gevent.pywsgi.WSGIServer', new=_define_mock_server(_raise_custom_error) ), seven.TemporaryDirectory() as temp_dir: recon_repo = ReconstructableRepository.from_yaml( file_relative_path(__file__, './repository.yaml') ) with pytest.raises(AnException): host_dagit_ui_with_reconstructable_repo( storage_fallback=temp_dir, recon_repo=recon_repo, host=None, port=2343 )
def test_port_collision(): def _raise_os_error(): raise OSError('Address already in use') with mock.patch( 'gevent.pywsgi.WSGIServer', new=_define_mock_server( _raise_os_error)), seven.TemporaryDirectory() as temp_dir: recon_repo = ReconstructableRepository.from_yaml( file_relative_path(__file__, './repository.yaml')) with pytest.raises(Exception) as exc_info: host_dagit_ui_with_reconstructable_repo( storage_fallback=temp_dir, recon_repo=recon_repo, host=None, port=2343, port_lookup=False, ) assert 'another instance of dagit ' in str(exc_info.value)
def test_execute_pipeline_with_ipc(): with seven.TemporaryDirectory() as temp_dir: instance = DagsterInstance.local_temp(temp_dir) events = [] for event in api_execute_pipeline( instance, ReconstructableRepository.from_yaml( file_relative_path(__file__, 'repository_file.yaml')), 'foo', {}, 'default', None, ): events.append(event) assert len(events) == 11 assert events[ 0].event_type_value == DagsterEventType.PIPELINE_START.value assert events[ -1].event_type_value == DagsterEventType.PIPELINE_SUCCESS.value
def test_deprecated_load(): repo = ReconstructableRepository.from_yaml( file_relative_path(__file__, 'repository.yaml')).get_definition() assert len(repo.get_all_pipelines()) == 1 assert len(repo.schedule_defs) == 1 assert len(repo.partition_set_defs) == 1
def test_repo_construction(): repo_yaml = file_relative_path(__file__, 'repo.yaml') assert ReconstructableRepository.from_yaml(repo_yaml).get_definition()
def get_main_external_repo(): return InProcessRepositoryLocation( ReconstructableRepository.from_yaml( file_relative_path(__file__, 'repo.yaml')), ).get_repository('test')
def get_main_recon_repo(): return ReconstructableRepository.from_yaml( file_relative_path(__file__, 'repo.yaml'))
def recon_pipeline_for_cli_args(kwargs): '''Builds a ReconstructablePipeline for CLI arguments, which can be any of the combinations for repo/pipeline loading above. ''' check.dict_param(kwargs, 'kwargs') pipeline_name = kwargs.get('pipeline_name') if pipeline_name and not isinstance(pipeline_name, six.string_types): if len(pipeline_name) == 1: pipeline_name = pipeline_name[0] else: check.failed( 'Can only handle zero or one pipeline args. Got {pipeline_name}'.format( pipeline_name=repr(pipeline_name) ) ) # Pipeline from repository YAML and pipeline_name if ( pipeline_name and kwargs.get('module_name') is None and kwargs.get('python_file') is None and kwargs.get('repository_yaml') is not None ): _cli_load_invariant(kwargs.get('fn_name') is None) repo_yaml = ( os.path.abspath(kwargs.get('repository_yaml')) if kwargs.get('repository_yaml') else DEFAULT_REPOSITORY_YAML_FILENAME ) _cli_load_invariant( os.path.exists(repo_yaml), 'Expected to use file "{}" to load repository but it does not exist. ' 'Verify your current working directory or CLI arguments.'.format(repo_yaml), ) return ReconstructableRepository.from_yaml(repo_yaml).get_reconstructable_pipeline( pipeline_name ) # Pipeline from repository python file elif kwargs.get('python_file') and kwargs.get('fn_name') and pipeline_name: _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('module_name') is None) return ReconstructableRepository.for_file( os.path.abspath(kwargs['python_file']), kwargs['fn_name'] ).get_reconstructable_pipeline(pipeline_name) # Pipeline from repository module elif kwargs.get('module_name') and kwargs.get('fn_name') and pipeline_name: _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('python_file') is None) return ReconstructableRepository.for_module( kwargs['module_name'], kwargs['fn_name'] ).get_reconstructable_pipeline(pipeline_name) # Pipeline from pipeline python file elif kwargs.get('python_file') and kwargs.get('fn_name') and not pipeline_name: _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('module_name') is None) return ReconstructablePipeline.for_file( os.path.abspath(kwargs['python_file']), kwargs['fn_name'] ) # Pipeline from pipeline module elif kwargs.get('module_name') and kwargs.get('fn_name') and not pipeline_name: _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('python_file') is None) return ReconstructablePipeline.for_module(kwargs['module_name'], kwargs['fn_name']) else: _cli_load_invariant(False)
def test_create_app_with_reconstructable_repo(): recon_repo = ReconstructableRepository.from_yaml( file_relative_path(__file__, './repository.yaml')) assert create_app_with_reconstructable_repo(recon_repo, DagsterInstance.ephemeral())
def define_context_for_repository_yaml(path, instance): check.inst_param(instance, 'instance', DagsterInstance) return DagsterGraphQLContext( locations=[InProcessRepositoryLocation(ReconstructableRepository.from_yaml(path))], instance=instance, )