def test_create_app_with_external_repo_data():
recon_repo = ReconstructableRepository.from_yaml(
file_relative_path(__file__, './repository.yaml'))
assert create_app_with_external_repository(
ExternalRepository.from_repository_def(recon_repo.get_definition()),
DagsterInstance.ephemeral(),
)
def define_context_for_repository_yaml(path, instance):
check.inst_param(instance, 'instance', DagsterInstance)
return DagsterGraphQLInProcessRepositoryContext(
recon_repo=ReconstructableRepository.from_yaml(path),
instance=instance,
execution_manager=SynchronousExecutionManager(),
)
def recon_repo_for_cli_args(kwargs):
'''Builds a ReconstructableRepository for CLI arguments, which can be any of the combinations
for repo loading above.
'''
check.dict_param(kwargs, 'kwargs')
_cli_load_invariant(kwargs.get('pipeline_name') is None)
if kwargs.get('repository_yaml') or all_none(kwargs):
_cli_load_invariant(kwargs.get('module_name') is None)
_cli_load_invariant(kwargs.get('python_file') is None)
_cli_load_invariant(kwargs.get('fn_name') is None)
repo_yaml = (
os.path.abspath(kwargs.get('repository_yaml'))
if kwargs.get('repository_yaml')
else DEFAULT_REPOSITORY_YAML_FILENAME
)
_cli_load_invariant(
os.path.exists(repo_yaml),
'Expected to use file "{}" to load repository but it does not exist. '
'Verify your current working directory or CLI arguments.'.format(repo_yaml),
)
return ReconstructableRepository.from_yaml(repo_yaml)
elif kwargs.get('module_name') and kwargs.get('fn_name'):
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('python_file') is None)
return ReconstructableRepository.for_module(kwargs['module_name'], kwargs['fn_name'])
elif kwargs.get('python_file') and kwargs.get('fn_name'):
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('module_name') is None)
return ReconstructableRepository.for_file(
os.path.abspath(kwargs['python_file']), kwargs['fn_name']
)
else:
_cli_load_invariant(False)
def test_successful_host_dagit_ui():
with mock.patch('gevent.pywsgi.WSGIServer'), seven.TemporaryDirectory() as temp_dir:
recon_repo = ReconstructableRepository.from_yaml(
file_relative_path(__file__, './repository.yaml')
)
host_dagit_ui_with_reconstructable_repo(
storage_fallback=temp_dir, recon_repo=recon_repo, host=None, port=2343
)
def test_index_view():
with create_app_with_reconstructable_repo(
ReconstructableRepository.from_yaml(file_relative_path(__file__, './repository.yaml')),
DagsterInstance.ephemeral(),
).test_client() as client:
res = client.get('/')
assert res.status_code == 200, res.data
assert b'You need to enable JavaScript to run this app' in res.data
def get_full_external_pipeline(repo_yaml, pipeline_name):
recon_repo = ReconstructableRepository.from_yaml(repo_yaml)
repo_def = recon_repo.get_definition()
external_repo = external_repo_from_def(
repo_def,
RepositoryHandle(
repo_def.name,
LocationHandle('<<MOCK_FOR_TEST>>', recon_repo.pointer),
),
)
return external_repo.get_full_external_pipeline(pipeline_name)
def define_context_for_repository_yaml(path, instance):
check.inst_param(instance, 'instance', DagsterInstance)
return DagsterGraphQLContext(
environments=[
InProcessDagsterEnvironment(
ReconstructableRepository.from_yaml(path),
execution_manager=SynchronousExecutionManager(),
)
],
instance=instance,
)
def test_notebook_view():
notebook_path = file_relative_path(__file__, 'render_uuid_notebook.ipynb')
with create_app_with_reconstructable_repo(
ReconstructableRepository.from_yaml(file_relative_path(__file__, './repository.yaml')),
DagsterInstance.ephemeral(),
).test_client() as client:
res = client.get('/dagit/notebook?path={}'.format(notebook_path))
assert res.status_code == 200
# This magic guid is hardcoded in the notebook
assert b'6cac0c38-2c97-49ca-887c-4ac43f141213' in res.data
def test_unknown_error():
class AnException(Exception):
pass
def _raise_custom_error():
raise AnException('foobar')
with mock.patch(
'gevent.pywsgi.WSGIServer', new=_define_mock_server(_raise_custom_error)
), seven.TemporaryDirectory() as temp_dir:
recon_repo = ReconstructableRepository.from_yaml(
file_relative_path(__file__, './repository.yaml')
)
with pytest.raises(AnException):
host_dagit_ui_with_reconstructable_repo(
storage_fallback=temp_dir, recon_repo=recon_repo, host=None, port=2343
)
def test_port_collision():
def _raise_os_error():
raise OSError('Address already in use')
with mock.patch(
'gevent.pywsgi.WSGIServer', new=_define_mock_server(
_raise_os_error)), seven.TemporaryDirectory() as temp_dir:
recon_repo = ReconstructableRepository.from_yaml(
file_relative_path(__file__, './repository.yaml'))
with pytest.raises(Exception) as exc_info:
host_dagit_ui_with_reconstructable_repo(
storage_fallback=temp_dir,
recon_repo=recon_repo,
host=None,
port=2343,
port_lookup=False,
)
assert 'another instance of dagit ' in str(exc_info.value)
def test_execute_pipeline_with_ipc():
with seven.TemporaryDirectory() as temp_dir:
instance = DagsterInstance.local_temp(temp_dir)
events = []
for event in api_execute_pipeline(
instance,
ReconstructableRepository.from_yaml(
file_relative_path(__file__, 'repository_file.yaml')),
'foo',
{},
'default',
None,
):
events.append(event)
assert len(events) == 11
assert events[
0].event_type_value == DagsterEventType.PIPELINE_START.value
assert events[
-1].event_type_value == DagsterEventType.PIPELINE_SUCCESS.value
def test_deprecated_load():
repo = ReconstructableRepository.from_yaml(
file_relative_path(__file__, 'repository.yaml')).get_definition()
assert len(repo.get_all_pipelines()) == 1
assert len(repo.schedule_defs) == 1
assert len(repo.partition_set_defs) == 1
def test_repo_construction():
repo_yaml = file_relative_path(__file__, 'repo.yaml')
assert ReconstructableRepository.from_yaml(repo_yaml).get_definition()
def get_main_external_repo():
return InProcessRepositoryLocation(
ReconstructableRepository.from_yaml(
file_relative_path(__file__,
'repo.yaml')), ).get_repository('test')
def get_main_recon_repo():
return ReconstructableRepository.from_yaml(
file_relative_path(__file__, 'repo.yaml'))
def recon_pipeline_for_cli_args(kwargs):
'''Builds a ReconstructablePipeline for CLI arguments, which can be
any of the combinations for repo/pipeline loading above.
'''
check.dict_param(kwargs, 'kwargs')
pipeline_name = kwargs.get('pipeline_name')
if pipeline_name and not isinstance(pipeline_name, six.string_types):
if len(pipeline_name) == 1:
pipeline_name = pipeline_name[0]
else:
check.failed(
'Can only handle zero or one pipeline args. Got {pipeline_name}'.format(
pipeline_name=repr(pipeline_name)
)
)
# Pipeline from repository YAML and pipeline_name
if (
pipeline_name
and kwargs.get('module_name') is None
and kwargs.get('python_file') is None
and kwargs.get('repository_yaml') is not None
):
_cli_load_invariant(kwargs.get('fn_name') is None)
repo_yaml = (
os.path.abspath(kwargs.get('repository_yaml'))
if kwargs.get('repository_yaml')
else DEFAULT_REPOSITORY_YAML_FILENAME
)
_cli_load_invariant(
os.path.exists(repo_yaml),
'Expected to use file "{}" to load repository but it does not exist. '
'Verify your current working directory or CLI arguments.'.format(repo_yaml),
)
return ReconstructableRepository.from_yaml(repo_yaml).get_reconstructable_pipeline(
pipeline_name
)
# Pipeline from repository python file
elif kwargs.get('python_file') and kwargs.get('fn_name') and pipeline_name:
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('module_name') is None)
return ReconstructableRepository.for_file(
os.path.abspath(kwargs['python_file']), kwargs['fn_name']
).get_reconstructable_pipeline(pipeline_name)
# Pipeline from repository module
elif kwargs.get('module_name') and kwargs.get('fn_name') and pipeline_name:
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('python_file') is None)
return ReconstructableRepository.for_module(
kwargs['module_name'], kwargs['fn_name']
).get_reconstructable_pipeline(pipeline_name)
# Pipeline from pipeline python file
elif kwargs.get('python_file') and kwargs.get('fn_name') and not pipeline_name:
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('module_name') is None)
return ReconstructablePipeline.for_file(
os.path.abspath(kwargs['python_file']), kwargs['fn_name']
)
# Pipeline from pipeline module
elif kwargs.get('module_name') and kwargs.get('fn_name') and not pipeline_name:
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('python_file') is None)
return ReconstructablePipeline.for_module(kwargs['module_name'], kwargs['fn_name'])
else:
_cli_load_invariant(False)
def test_create_app_with_reconstructable_repo():
recon_repo = ReconstructableRepository.from_yaml(
file_relative_path(__file__, './repository.yaml'))
assert create_app_with_reconstructable_repo(recon_repo,
DagsterInstance.ephemeral())
def define_context_for_repository_yaml(path, instance):
check.inst_param(instance, 'instance', DagsterInstance)
return DagsterGraphQLContext(
locations=[InProcessRepositoryLocation(ReconstructableRepository.from_yaml(path))],
instance=instance,
)
