def test_workspace_yamls():
with load_workspace_process_context_from_yaml_paths(
DagsterInstance.ephemeral(),
[
file_relative_path(
__file__,
"../../../docs_snippets/concepts/repositories_workspaces/workspace.yaml",
)
],
) as workspace_process_context:
assert workspace_process_context.repository_locations_count == 1
with load_workspace_process_context_from_yaml_paths(
DagsterInstance.ephemeral(),
[
file_relative_path(
__file__,
"../../../docs_snippets/concepts/repositories_workspaces/workspace_working_directory.yaml",
)
],
) as workspace_process_context:
assert workspace_process_context.repository_locations_count == 2
with load_workspace_process_context_from_yaml_paths(
DagsterInstance.ephemeral(),
[
file_relative_path(
__file__,
"../../../docs_snippets/concepts/repositories_workspaces/workspace_one_repository.yaml",
)
],
) as workspace_process_context:
assert workspace_process_context.repository_locations_count == 1
with load_workspace_process_context_from_yaml_paths(
DagsterInstance.ephemeral(),
[
file_relative_path(
__file__,
"../../../docs_snippets/concepts/repositories_workspaces/workspace_python_package.yaml",
)
],
) as workspace_process_context:
assert workspace_process_context.repository_locations_count == 1
with load_workspace_process_context_from_yaml_paths(
DagsterInstance.ephemeral(),
[
file_relative_path(
__file__,
"../../../docs_snippets/concepts/repositories_workspaces/workspace_grpc.yaml",
)
],
) as workspace_process_context:
assert workspace_process_context.repository_locations_count == 1
def test_invalid_path_prefix():
with mock.patch("uvicorn.run"), tempfile.TemporaryDirectory() as temp_dir:
instance = DagsterInstance.local_temp(temp_dir)
with load_workspace_process_context_from_yaml_paths(
instance, [file_relative_path(__file__, "./workspace.yaml")
]) as workspace_process_context:
with pytest.raises(Exception) as exc_info:
host_dagit_ui_with_workspace_process_context(
workspace_process_context=workspace_process_context,
host=None,
port=2343,
path_prefix="no-leading-slash",
)
assert "path prefix should begin with a leading" in str(
exc_info.value)
with pytest.raises(Exception) as exc_info:
host_dagit_ui_with_workspace_process_context(
workspace_process_context=workspace_process_context,
host=None,
port=2343,
path_prefix="/extra-trailing-slash/",
)
assert "path prefix should not include a trailing" in str(
exc_info.value)
def test_create_app_with_workspace():
with load_workspace_process_context_from_yaml_paths(
DagsterInstance.ephemeral(),
[file_relative_path(__file__, "./workspace.yaml")],
) as workspace_process_context:
assert create_app_from_workspace_process_context(
workspace_process_context)
def test_load_in_process_location_hello_world_terse():
with load_workspace_process_context_from_yaml_paths(
DagsterInstance.ephemeral(),
[file_relative_path(__file__, "terse_python_module_workspace.yaml")],
) as workspace:
assert isinstance(workspace, WorkspaceProcessContext)
assert workspace.repository_locations_count == 1
assert workspace.repository_location_names[0] == "dagster.utils.test.hello_world_repository"
def test_graphql_view_at_path_prefix():
with load_workspace_process_context_from_yaml_paths(
DagsterInstance.ephemeral(), [file_relative_path(__file__, "./workspace.yaml")]
) as workspace_process_context:
with create_app_from_workspace_process_context(
workspace_process_context, "/dagster-path"
).test_client() as client:
res = client.get("/dagster-path/graphql")
assert b"Must provide query string" in res.data
def test_multi_location_workspace_foo():
with load_workspace_process_context_from_yaml_paths(
DagsterInstance.ephemeral(),
[file_relative_path(__file__, "multi_location.yaml")],
) as grpc_workspace:
assert isinstance(grpc_workspace, WorkspaceProcessContext)
assert grpc_workspace.repository_locations_count == 3
assert grpc_workspace.has_repository_location("loaded_from_file")
assert grpc_workspace.has_repository_location("loaded_from_module")
assert grpc_workspace.has_repository_location("loaded_from_package")
def test_graphql_view():
with load_workspace_process_context_from_yaml_paths(
DagsterInstance.ephemeral(),
[file_relative_path(__file__, "./workspace.yaml")
]) as workspace_process_context:
client = TestClient(
create_app_from_workspace_process_context(
workspace_process_context, ))
res = client.get("/graphql")
assert b"No GraphQL query found in the request" in res.content
def test_load_in_process_location_hello_world_nested_with_def():
file_name = file_relative_path(
__file__, "nested_with_def_python_file_workspace.yaml")
with load_workspace_process_context_from_yaml_paths(
DagsterInstance.ephemeral(),
[file_name]) as workspace_process_context:
assert isinstance(workspace_process_context, WorkspaceProcessContext)
assert workspace_process_context.repository_locations_count == 1
assert (workspace_process_context.repository_location_names[0] ==
"hello_world_repository.py:hello_world_repository")
def test_index_view():
with load_workspace_process_context_from_yaml_paths(
DagsterInstance.ephemeral(), [file_relative_path(__file__, "./workspace.yaml")]
) as workspace_process_context:
with create_app_from_workspace_process_context(
workspace_process_context
).test_client() as client:
res = client.get("/")
assert res.status_code == 200, res.data
assert b"You need to enable JavaScript to run this app" in res.data
def test_successful_host_dagit_ui_from_legacy_repository():
with mock.patch("uvicorn.run"), tempfile.TemporaryDirectory() as temp_dir:
instance = DagsterInstance.local_temp(temp_dir)
with load_workspace_process_context_from_yaml_paths(
instance, [file_relative_path(__file__, "./workspace.yaml")
]) as workspace_process_context:
host_dagit_ui_with_workspace_process_context(
workspace_process_context=workspace_process_context,
host=None,
port=2343,
path_prefix="",
)
def test_log_workspace_stats(caplog):
with instance_for_test(overrides={"telemetry": {"enabled": True}}) as instance:
with load_workspace_process_context_from_yaml_paths(
instance, [file_relative_path(__file__, "./multi_env_telemetry_workspace.yaml")]
) as context:
log_workspace_stats(instance, context)
for record in caplog.records:
message = json.loads(record.getMessage())
assert message.get("action") == UPDATE_REPO_STATS
assert set(message.keys()) == EXPECTED_KEYS
assert len(caplog.records) == 2
def test_bad_workspace_yaml_load():
with TemporaryDirectory() as temp_dir:
touch_file(os.path.join(temp_dir, "foo.yaml"))
with pytest.raises(
CheckError,
match=
("Invariant failed. Description: Could not parse a workspace config from the "
"yaml file at"),
):
with load_workspace_process_context_from_yaml_paths(
DagsterInstance.ephemeral(),
[os.path.join(temp_dir, "foo.yaml")]):
pass
def test_notebook_view():
notebook_path = file_relative_path(__file__, "render_uuid_notebook.ipynb")
with load_workspace_process_context_from_yaml_paths(
DagsterInstance.ephemeral(), [file_relative_path(__file__, "./workspace.yaml")]
) as workspace_process_context:
with create_app_from_workspace_process_context(
workspace_process_context,
).test_client() as client:
res = client.get(f"/dagit/notebook?path={notebook_path}&repoLocName=load_from_file")
assert res.status_code == 200
# This magic guid is hardcoded in the notebook
assert b"6cac0c38-2c97-49ca-887c-4ac43f141213" in res.data
def test_multi_file_extend_workspace():
with load_workspace_process_context_from_yaml_paths(
DagsterInstance.ephemeral(),
[
file_relative_path(__file__, "multi_location.yaml"),
file_relative_path(__file__, "extra_location.yaml"),
],
) as workspace:
assert isinstance(workspace, WorkspaceProcessContext)
assert workspace.repository_locations_count == 4
assert workspace.has_repository_location("loaded_from_file")
assert workspace.has_repository_location("loaded_from_module")
assert workspace.has_repository_location("loaded_from_package")
assert workspace.has_repository_location("extra_location")
def test_valid_path_prefix():
with mock.patch("gevent.pywsgi.WSGIServer"), tempfile.TemporaryDirectory() as temp_dir:
instance = DagsterInstance.local_temp(temp_dir)
with load_workspace_process_context_from_yaml_paths(
instance, [file_relative_path(__file__, "./workspace.yaml")]
) as workspace_process_context:
host_dagit_ui_with_workspace_process_context(
workspace_process_context=workspace_process_context,
host=None,
port=2343,
port_lookup=False,
path_prefix="/dagster-path",
)
def test_index_view_at_path_prefix():
with load_workspace_process_context_from_yaml_paths(
DagsterInstance.ephemeral(), [file_relative_path(__file__, "./workspace.yaml")]
) as workspace_process_context:
with create_app_from_workspace_process_context(
workspace_process_context, "/dagster-path"
).test_client() as client:
# / redirects to prefixed path
res = client.get("/")
assert res.status_code == 200
# index contains the path meta tag
res = client.get("/dagster-path/")
assert res.status_code == 200
assert b"You need to enable JavaScript to run this app" in res.data
assert b'{"pathPrefix": "/dagster-path"}' in res.data
def test_create_app_with_workspace_and_scheduler():
with tempfile.TemporaryDirectory() as temp_dir:
with instance_for_test(
temp_dir=temp_dir,
overrides={
"scheduler": {
"module": "dagster.utils.test",
"class": "FilesystemTestScheduler",
"config": {"base_dir": temp_dir},
}
},
) as instance:
with load_workspace_process_context_from_yaml_paths(
instance, [file_relative_path(__file__, "./workspace.yaml")]
) as workspace_process_context:
assert create_app_from_workspace_process_context(workspace_process_context)
def test_successful_host_dagit_ui_from_multiple_workspace_files():
with mock.patch("gevent.pywsgi.WSGIServer"), tempfile.TemporaryDirectory() as temp_dir:
instance = DagsterInstance.local_temp(temp_dir)
with load_workspace_process_context_from_yaml_paths(
instance,
[
file_relative_path(__file__, "./workspace.yaml"),
file_relative_path(__file__, "./override.yaml"),
],
) as workspace_process_context:
host_dagit_ui_with_workspace_process_context(
workspace_process_context=workspace_process_context,
host=None,
port=2343,
path_prefix="",
)
def test_workspace_with_only_error():
with load_workspace_process_context_from_yaml_paths(
DagsterInstance.ephemeral(),
[file_relative_path(__file__, "workspace_with_only_error.yaml")],
) as cli_workspace:
assert isinstance(cli_workspace, WorkspaceProcessContext)
assert cli_workspace.repository_locations_count == 1
assert not cli_workspace.has_repository_location("broken_location")
request_context = cli_workspace.create_request_context()
assert len(request_context.repository_location_errors()) == 1
assert request_context.has_repository_location_error("broken_location")
assert (
"No module named"
in request_context.get_repository_location_error("broken_location").message
)
def test_unknown_error():
class AnException(Exception):
pass
def _raise_custom_error():
raise AnException("foobar")
with mock.patch(
"gevent.pywsgi.WSGIServer", new=_define_mock_server(_raise_custom_error)
), tempfile.TemporaryDirectory() as temp_dir:
instance = DagsterInstance.local_temp(temp_dir)
with load_workspace_process_context_from_yaml_paths(
instance, [file_relative_path(__file__, "./workspace.yaml")]
) as workspace_process_context:
with pytest.raises(AnException):
host_dagit_ui_with_workspace_process_context(
workspace_process_context=workspace_process_context,
host=None,
port=2343,
path_prefix="",
)
def test_multi_file_override_workspace():
with load_workspace_process_context_from_yaml_paths(
DagsterInstance.ephemeral(),
[
file_relative_path(__file__, "multi_location.yaml"),
file_relative_path(__file__, "override_location.yaml"),
],
) as workspace:
assert isinstance(workspace, WorkspaceProcessContext)
assert workspace.repository_locations_count == 3
assert workspace.has_repository_location("loaded_from_file")
assert workspace.has_repository_location("loaded_from_module")
assert workspace.has_repository_location("loaded_from_package")
loaded_from_file = workspace.create_request_context(
).get_repository_location("loaded_from_file")
# Ensure location `loaded_from_file` has been overridden
external_repositories = loaded_from_file.get_repositories()
assert len(external_repositories) == 1
assert "extra_repository" in external_repositories
def test_port_collision():
def _raise_os_error():
raise OSError("Address already in use")
with mock.patch(
"gevent.pywsgi.WSGIServer", new=_define_mock_server(_raise_os_error)
), tempfile.TemporaryDirectory() as temp_dir:
instance = DagsterInstance.local_temp(temp_dir)
with load_workspace_process_context_from_yaml_paths(
instance, [file_relative_path(__file__, "./workspace.yaml")]
) as workspace_process_context:
with pytest.raises(Exception) as exc_info:
host_dagit_ui_with_workspace_process_context(
workspace_process_context=workspace_process_context,
host=None,
port=2343,
port_lookup=False,
path_prefix="",
)
assert "another instance of dagit " in str(exc_info.value)
