def test_existing_local_data_docs_urls_returns_single_url_from_customized_local_site(
tmp_path_factory, ):
empty_directory = str(tmp_path_factory.mktemp("yo_yo"))
DataContext.create(empty_directory)
ge_dir = os.path.join(empty_directory, DataContext.GE_DIR)
context = DataContext(ge_dir)
context._project_config["data_docs_sites"] = {
"my_rad_site": {
"class_name": "SiteBuilder",
"store_backend": {
"class_name": "TupleFilesystemStoreBackend",
"base_directory": "uncommitted/data_docs/some/local/path/",
},
}
}
# TODO Workaround project config programmatic config manipulation
# statefulness issues by writing to disk and re-upping a new context
context._save_project_config()
context = DataContext(ge_dir)
context.build_data_docs()
expected_path = os.path.join(
ge_dir, "uncommitted/data_docs/some/local/path/index.html")
assert os.path.isfile(expected_path)
obs = context.get_docs_sites_urls()
assert obs == [{
"site_name": "my_rad_site",
"site_url": "file://{}".format(expected_path)
}]
def test_docs_build_view(
mock_webbrowser, caplog,
site_builder_data_context_with_html_store_titanic_random):
root_dir = site_builder_data_context_with_html_store_titanic_random.root_directory
runner = CliRunner(mix_stderr=False)
result = runner.invoke(cli, ["docs", "build", "-d", root_dir],
input="\n\n",
catch_exceptions=False)
stdout = result.stdout
assert result.exit_code == 0
assert mock_webbrowser.call_count == 1
assert "Building" in stdout
assert "The following Data Docs sites will be built:" in stdout
assert "great_expectations/uncommitted/data_docs/local_site/index.html" in stdout
context = DataContext(root_dir)
obs_urls = context.get_docs_sites_urls()
assert len(obs_urls) == 2
assert ("great_expectations/uncommitted/data_docs/local_site/index.html"
in obs_urls[0]["site_url"])
local_site_dir = os.path.join(root_dir,
"uncommitted/data_docs/local_site/")
assert os.path.isdir(os.path.join(local_site_dir))
assert os.path.isfile(os.path.join(local_site_dir, "index.html"))
assert os.path.isdir(os.path.join(local_site_dir, "expectations"))
assert os.path.isdir(os.path.join(local_site_dir, "validations"))
assert_no_logging_messages_or_tracebacks(
my_caplog=caplog,
click_result=result,
)
def test_suite_demo_multiple_datasources_with_generator_with_suite_name_argument(
mock_webbrowser,
mock_subprocess,
caplog,
site_builder_data_context_with_html_store_titanic_random,
):
"""
We call the "suite demo" command with the suite name argument
- The data context has two datasources - we choose one of them.
- It has a generator configured. We choose to use the generator and select
a generator asset from the list.
- open Data Docs
- NOT open jupyter
"""
root_dir = site_builder_data_context_with_html_store_titanic_random.root_directory
os.chdir(root_dir)
context = DataContext(root_dir)
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli,
["suite", "demo", "-d", root_dir, "--suite", "foo_suite"],
input="2\n1\n1\n\n",
catch_exceptions=False,
)
stdout = result.stdout
assert result.exit_code == 0
assert "Select a datasource" in stdout
assert "Which data would you like to use" in stdout
assert (
"Great Expectations will choose a couple of columns and generate expectations"
in stdout
)
assert "Generating example Expectation Suite..." in stdout
assert "Building" in stdout
assert "The following Data Docs sites were built" in stdout
assert "A new Expectation suite 'foo_suite' was added to your project" in stdout
obs_urls = context.get_docs_sites_urls()
assert len(obs_urls) == 1
assert (
"great_expectations/uncommitted/data_docs/local_site/index.html"
in obs_urls[0]["site_url"]
)
expected_index_path = os.path.join(
root_dir, "uncommitted", "data_docs", "local_site", "index.html"
)
assert os.path.isfile(expected_index_path)
expected_suite_path = os.path.join(root_dir, "expectations", "foo_suite.json")
assert os.path.isfile(expected_suite_path)
assert mock_webbrowser.call_count == 1
assert mock_subprocess.call_count == 0
assert_no_logging_messages_or_tracebacks(caplog, result)
def test_suite_new_multiple_datasources_with_generator_without_suite_name_argument(
caplog, site_builder_data_context_with_html_store_titanic_random,
):
"""
We call the "suite new" command without the suite name argument
The data context has two datasources - we choose one of them. It has a generator
configured. We choose to use the generator and select a generator asset from the list.
The command should prompt us to enter the name of the expectation suite that will be
created.
"""
root_dir = site_builder_data_context_with_html_store_titanic_random.root_directory
os.chdir(root_dir)
context = DataContext(root_dir)
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli,
["suite", "new", "-d", root_dir, "--no-view"],
input="1\n1\n1\nmy_new_suite\n\n",
catch_exceptions=False,
)
stdout = result.stdout
assert result.exit_code == 0
assert """Select a datasource
1. random
2. titanic""" in stdout
assert """Which data would you like to use?
1. f1 (file)
2. f2 (file)""" in stdout
assert "Name the new expectation suite [f1.warning]" in stdout
assert (
"Great Expectations will choose a couple of columns and generate expectations"
in stdout
)
assert "Generating example Expectation Suite..." in stdout
assert "Building" in stdout
assert "The following Data Docs sites were built" in stdout
assert "A new Expectation suite 'my_new_suite' was added to your project" in stdout
obs_urls = context.get_docs_sites_urls()
assert len(obs_urls) == 1
assert (
"great_expectations/uncommitted/data_docs/local_site/index.html" in obs_urls[0]
)
expected_index_path = os.path.join(
root_dir, "uncommitted", "data_docs", "local_site", "index.html"
)
assert os.path.isfile(expected_index_path)
expected_suite_path = os.path.join(root_dir, "expectations", "my_new_suite.json")
assert os.path.isfile(expected_suite_path)
assert_no_logging_messages_or_tracebacks(caplog, result)
def test_existing_local_data_docs_urls_returns_url_on_project_with_no_datasources_and_a_site_configured(
tmp_path_factory, ):
"""
This test ensures that a url will be returned for a default site even if a
datasource is not configured, and docs are not built.
"""
empty_directory = str(tmp_path_factory.mktemp("another_empty_project"))
DataContext.create(empty_directory)
context = DataContext(os.path.join(empty_directory, DataContext.GE_DIR))
obs = context.get_docs_sites_urls(only_if_exists=False)
assert len(obs) == 1
assert obs[0]["site_url"].endswith(
"great_expectations/uncommitted/data_docs/local_site/index.html")
def test_docs_build_no_view(
mock_webbrowser,
caplog,
monkeypatch,
site_builder_data_context_v013_with_html_store_titanic_random,
):
context = site_builder_data_context_v013_with_html_store_titanic_random
root_dir = context.root_directory
runner = CliRunner(mix_stderr=False)
monkeypatch.chdir(os.path.dirname(context.root_directory))
result = runner.invoke(
cli,
[
"--v3-api",
"docs",
"build",
"--no-view",
],
input="\n",
catch_exceptions=False,
)
stdout = result.stdout
assert result.exit_code == 0
assert mock_webbrowser.call_count == 0
assert "Building" in stdout
assert "The following Data Docs sites will be built:" in stdout
assert "great_expectations/uncommitted/data_docs/local_site/index.html" in stdout
context = DataContext(root_dir)
obs_urls = context.get_docs_sites_urls()
assert len(obs_urls) == 2
assert (
"great_expectations/uncommitted/data_docs/local_site/index.html"
in obs_urls[0]["site_url"]
)
local_site_dir = os.path.join(root_dir, "uncommitted/data_docs/local_site/")
assert os.path.isdir(os.path.join(local_site_dir))
assert os.path.isfile(os.path.join(local_site_dir, "index.html"))
assert os.path.isdir(os.path.join(local_site_dir, "expectations"))
assert os.path.isdir(os.path.join(local_site_dir, "validations"))
assert_no_logging_messages_or_tracebacks(
my_caplog=caplog,
click_result=result,
allowed_deprecation_message=VALIDATION_OPERATORS_DEPRECATION_MESSAGE,
)
def docs_list(directory):
"""List known Data Docs Sites."""
context = None
try:
failed = True
context = DataContext(directory)
docs_sites_url_dicts = context.get_docs_sites_urls()
docs_sites_strings = [
" - <cyan>{}</cyan>: {}".format(docs_site_dict["site_name"], docs_site_dict["site_url"])\
for docs_site_dict in docs_sites_url_dicts
]
if len(docs_sites_strings) == 0:
cli_message("No Data Docs sites found")
failed = False
send_usage_message(data_context=context,
event="cli.docs.list",
success=True)
return
if len(docs_sites_strings) == 1:
list_intro_string = "1 Data Docs site found:"
if len(docs_sites_strings) > 1:
list_intro_string = "{} Data Docs sites found:".format(
len(docs_sites_strings))
cli_message_list(docs_sites_strings, list_intro_string)
failed = False
send_usage_message(data_context=context,
event="cli.docs.list",
success=True)
except ge_exceptions.ConfigNotFoundError as err:
cli_message("<red>{}</red>".format(err.message))
finally:
if failed and context is not None:
send_usage_message(data_context=context,
event="cli.docs.list",
success=False)
def test_suite_demo_one_datasource_without_generator_without_suite_name_argument(
mock_webbrowser, mock_subprocess, caplog, empty_data_context, filesystem_csv_2
):
"""
We call the "suite demo" command without the suite name argument
The command should:
- NOT prompt us to choose a datasource (because there is only one)
- prompt us only to enter the path (The datasource has no generator
configured and not to choose from the generator's list of available data
assets).
- We enter the path of the file we want the command to use as the batch to
create the expectation suite.
- prompt us to enter the name of the expectation suite that will be
created
- open Data Docs
- NOT open jupyter
"""
empty_data_context.add_datasource(
"my_datasource",
module_name="great_expectations.datasource",
class_name="PandasDatasource",
)
context = empty_data_context
root_dir = context.root_directory
context = DataContext(root_dir)
runner = CliRunner(mix_stderr=False)
csv_path = os.path.join(filesystem_csv_2, "f1.csv")
result = runner.invoke(
cli,
["suite", "demo", "-d", root_dir],
input=f"{csv_path}\nmy_new_suite\n\n",
catch_exceptions=False,
)
stdout = result.stdout
assert result.exit_code == 0
assert "Enter the path" in stdout
assert "Name the new Expectation Suite [f1.warning]" in stdout
assert (
"Great Expectations will choose a couple of columns and generate expectations"
in stdout
)
assert (
"Great Expectations will store these expectations in a new Expectation Suite 'my_new_suite' here:"
in stdout
)
assert "Generating example Expectation Suite..." in stdout
assert "Building" in stdout
assert "The following Data Docs sites will be built" in stdout
obs_urls = context.get_docs_sites_urls()
assert len(obs_urls) == 1
assert (
"great_expectations/uncommitted/data_docs/local_site/index.html"
in obs_urls[0]["site_url"]
)
expected_index_path = os.path.join(
root_dir, "uncommitted", "data_docs", "local_site", "index.html"
)
assert os.path.isfile(expected_index_path)
expected_suite_path = os.path.join(root_dir, "expectations", "my_new_suite.json")
assert os.path.isfile(expected_suite_path)
assert mock_webbrowser.call_count == 1
assert mock_subprocess.call_count == 0
assert_no_logging_messages_or_tracebacks(caplog, result)
def test_suite_new_one_datasource_without_generator_without_suite_name_argument(
caplog, empty_data_context, filesystem_csv_2):
"""
We call the "suite new" command without the suite name argument
The data context has one datasource, so the command does not prompt us to choose.
The datasource has no generator configured, so we are prompted only to enter the path
(and not to choose from the generator's list of available data assets).
We enter the path of the file we want the command to use as the batch to create the
expectation suite.
The command should prompt us to enter the name of the expectation suite that will be
created.
"""
empty_data_context.add_datasource(
"my_datasource",
module_name="great_expectations.datasource",
class_name="PandasDatasource",
)
not_so_empty_data_context = empty_data_context
project_root_dir = not_so_empty_data_context.root_directory
root_dir = project_root_dir
os.chdir(root_dir)
context = DataContext(root_dir)
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli,
["suite", "new", "-d", root_dir, "--no-view"],
input="{0:s}\nmy_new_suite\n\n".format(
os.path.join(filesystem_csv_2, "f1.csv")),
catch_exceptions=False,
)
stdout = result.stdout
assert result.exit_code == 0
assert "Enter the path" in stdout
assert "Name the new expectation suite [warning]" in stdout
assert (
"Great Expectations will choose a couple of columns and generate expectations"
in stdout)
assert "Generating example Expectation Suite..." in stdout
assert "Building" in stdout
assert "The following Data Docs sites were built" in stdout
assert "A new Expectation suite 'my_new_suite' was added to your project" in stdout
obs_urls = context.get_docs_sites_urls()
assert len(obs_urls) == 1
assert ("great_expectations/uncommitted/data_docs/local_site/index.html"
in obs_urls[0])
expected_index_path = os.path.join(root_dir, "uncommitted", "data_docs",
"local_site", "index.html")
assert os.path.isfile(expected_index_path)
expected_suite_path = os.path.join(root_dir, "expectations",
"my_new_suite.json")
assert os.path.isfile(expected_suite_path)
assert_no_logging_messages_or_tracebacks(caplog, result)
def test_docs_build_view(
mock_webbrowser,
mock_emit,
caplog,
monkeypatch,
titanic_data_context_stats_enabled_config_version_3,
):
context = titanic_data_context_stats_enabled_config_version_3
root_dir = context.root_directory
runner = CliRunner(mix_stderr=False)
monkeypatch.chdir(os.path.dirname(context.root_directory))
result = runner.invoke(
cli,
"--v3-api docs build",
input="\n",
catch_exceptions=False,
)
stdout = result.stdout
assert result.exit_code == 0
assert mock_webbrowser.call_count == 1
assert "Would you like to proceed?" in stdout
assert "Building" in stdout
assert "The following Data Docs sites will be built:" in stdout
assert "local_site" in stdout
assert "great_expectations/uncommitted/data_docs/local_site/index.html" in stdout
assert mock_emit.call_count == 4
assert mock_emit.call_args_list == [
mock.call({
"event_payload": {},
"event": "data_context.__init__",
"success": True
}),
mock.call({
"event_payload": {},
"event": "data_context.build_data_docs",
"success": True,
}),
mock.call({
"event_payload": {},
"event": "data_context.open_data_docs",
"success": True,
}),
mock.call({
"event": "cli.docs.build",
"event_payload": {
"api_version": "v3"
},
"success": True,
}),
]
context = DataContext(root_dir)
obs_urls = context.get_docs_sites_urls()
assert len(obs_urls) == 1
assert ("great_expectations/uncommitted/data_docs/local_site/index.html"
in obs_urls[0]["site_url"])
site_dir = os.path.join(root_dir, context.GE_UNCOMMITTED_DIR, "data_docs",
"local_site")
assert os.path.isdir(site_dir)
# Note the fixture has no expectations or validations - only check the index
assert os.path.isfile(os.path.join(site_dir, "index.html"))
assert_no_logging_messages_or_tracebacks(
my_caplog=caplog,
click_result=result,
)
def test_docs_build_happy_paths_build_site_on_single_site_context(
mock_webbrowser,
mock_emit,
invocation,
cli_input,
expected_stdout,
expected_browser_call_count,
caplog,
monkeypatch,
titanic_data_context_stats_enabled_config_version_3,
):
context = titanic_data_context_stats_enabled_config_version_3
root_dir = context.root_directory
runner = CliRunner(mix_stderr=False)
monkeypatch.chdir(os.path.dirname(context.root_directory))
result = runner.invoke(
cli,
invocation,
input=cli_input,
catch_exceptions=False,
)
stdout = result.stdout
assert result.exit_code == 0
assert "The following Data Docs sites will be built" in stdout
assert "local_site" in stdout
if not cli_input == "n\n":
assert "Building Data Docs..." in stdout
assert "Done building Data Docs" in stdout
assert expected_stdout in stdout
assert mock_webbrowser.call_count == expected_browser_call_count
expected_usage_stats_messages = [
mock.call({
"event_payload": {},
"event": "data_context.__init__",
"success": True
}),
mock.call({
"event": "cli.docs.build.begin",
"event_payload": {
"api_version": "v3"
},
"success": True,
}),
]
if not cli_input == "n\n":
expected_usage_stats_messages.append(
mock.call({
"event_payload": {},
"event": "data_context.build_data_docs",
"success": True,
}), )
if expected_browser_call_count == 1:
expected_usage_stats_messages.append(
mock.call({
"event_payload": {},
"event": "data_context.open_data_docs",
"success": True,
}), )
build_docs_end_event_payload = {"api_version": "v3"}
if cli_input == "n\n":
build_docs_end_event_payload.update({"cancelled": True})
expected_usage_stats_messages.append(
mock.call({
"event": "cli.docs.build.end",
"event_payload": build_docs_end_event_payload,
"success": True,
}), )
assert mock_emit.call_args_list == expected_usage_stats_messages
context = DataContext(root_dir)
obs_urls = context.get_docs_sites_urls()
assert len(obs_urls) == 1
if not cli_input == "n\n":
assert (
"great_expectations/uncommitted/data_docs/local_site/index.html"
in obs_urls[0]["site_url"])
site_dir = os.path.join(root_dir, context.GE_UNCOMMITTED_DIR,
"data_docs", "local_site")
assert os.path.isdir(site_dir)
# Note the fixture has no expectations or validations - only check the index
assert os.path.isfile(os.path.join(site_dir, "index.html"))
assert_no_logging_messages_or_tracebacks(
my_caplog=caplog,
click_result=result,
)
