def test_init_on_existing_project_with_datasource_with_no_suite_create_one(
mock_browser,
caplog,
initialized_project,
):
project_dir = initialized_project
ge_dir = os.path.join(project_dir, DataContext.GE_DIR)
uncommitted_dir = os.path.join(ge_dir, "uncommitted")
data_folder_path = os.path.join(project_dir, "data")
data_path = os.path.join(project_dir, "data", "Titanic.csv")
# mangle the setup to remove all traces of any suite
expectations_dir = os.path.join(ge_dir, "expectations")
data_docs_dir = os.path.join(uncommitted_dir, "data_docs")
validations_dir = os.path.join(uncommitted_dir, "validations")
_delete_and_recreate_dir(expectations_dir)
_delete_and_recreate_dir(data_docs_dir)
_delete_and_recreate_dir(validations_dir)
context = DataContext(ge_dir)
assert context.list_expectation_suites() == []
runner = CliRunner(mix_stderr=False)
with pytest.warns(
UserWarning,
match="Warning. An existing `great_expectations.yml` was found"):
result = runner.invoke(
cli,
["init", "-d", project_dir],
input="\n2\n{}\nsink_me\n\n\n".format(data_path),
catch_exceptions=False,
)
stdout = result.stdout
assert result.exit_code == 0
assert mock_browser.call_count == 1
assert "Error: invalid input" not in stdout
assert "Always know what to expect from your data" in stdout
assert "Enter the path (relative or absolute) of a data file" in stdout
assert "Generating example Expectation Suite..." in stdout
assert "The following Data Docs sites will be built" in stdout
assert "Great Expectations is now set up" in stdout
assert_no_logging_messages_or_tracebacks(caplog, result)
def test_cli_datasource_list(empty_data_context, empty_sqlite_db, caplog):
"""Test an empty project and after adding a single datasource."""
project_root_dir = empty_data_context.root_directory
context = DataContext(project_root_dir)
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli, ["datasource", "list", "-d", project_root_dir], catch_exceptions=False
)
stdout = result.stdout.strip()
assert "No Datasources found" in stdout
assert context.list_datasources() == []
datasource_name = "wow_a_datasource"
_add_datasource_and_credentials_to_context(
context, datasource_name, empty_sqlite_db
)
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli, ["datasource", "list", "-d", project_root_dir], catch_exceptions=False
)
url = str(empty_sqlite_db.engine.url)
expected_output = """\
1 Datasource found:[0m
[0m
- [36mname:[0m wow_a_datasource[0m
[36mmodule_name:[0m great_expectations.datasource[0m
[36mclass_name:[0m SqlAlchemyDatasource[0m
[36mbatch_kwargs_generators:[0m[0m
[36mdefault:[0m[0m
[36mclass_name:[0m TableBatchKwargsGenerator[0m
[36mcredentials:[0m[0m
[36murl:[0m {}[0m
[36mdata_asset_type:[0m[0m
[36mclass_name:[0m SqlAlchemyDataset[0m
[36mmodule_name:[0m None[0m
""".format(
url
).strip()
stdout = result.stdout.strip()
assert stdout == expected_output
assert_no_logging_messages_or_tracebacks(caplog, result)
def context_with_two_sites(titanic_data_context_stats_enabled_config_version_3):
context = titanic_data_context_stats_enabled_config_version_3
config = context.get_config_with_variables_substituted()
config.data_docs_sites["team_site"] = {
"class_name": "SiteBuilder",
"store_backend": {
"class_name": "TupleFilesystemStoreBackend",
"base_directory": "uncommitted/data_docs/team_site/",
},
"site_index_builder": {"class_name": "DefaultSiteIndexBuilder"},
}
temp_context = BaseDataContext(config, context_root_dir=context.root_directory)
new_context = DataContext(context.root_directory)
new_context.set_config(temp_context.get_config_with_variables_substituted())
new_context._save_project_config()
assert new_context.get_site_names() == ["local_site", "team_site"]
return new_context
def test_validation_operator_list_with_zero_validation_operators(
caplog, empty_data_context):
project_dir = empty_data_context.root_directory
context = DataContext(project_dir)
context._project_config.validation_operators = {}
context._save_project_config()
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli,
"validation-operator list -d {}".format(project_dir),
catch_exceptions=False,
)
assert result.exit_code == 0
assert "No Validation Operators found" in result.output
assert_no_logging_messages_or_tracebacks(caplog, result)
def load_data_context_with_error_handling(
directory: str, from_cli_upgrade_command: bool = False
) -> DataContext:
"""Return a DataContext with good error handling and exit codes."""
# TODO consolidate all the myriad CLI tests into this
try:
context = DataContext(directory)
return context
except ge_exceptions.UnsupportedConfigVersionError as err:
directory = directory or DataContext.find_context_root_dir()
ge_config_version = DataContext.get_ge_config_version(
context_root_dir=directory
)
upgrade_helper_class = (
GE_UPGRADE_HELPER_VERSION_MAP.get(int(ge_config_version))
if ge_config_version
else None
)
if (
upgrade_helper_class
and ge_config_version < MINIMUM_SUPPORTED_CONFIG_VERSION
):
upgrade_project(
context_root_dir=directory,
ge_config_version=ge_config_version,
from_cli_upgrade_command=from_cli_upgrade_command,
)
else:
cli_message("<red>{}</red>".format(err.message))
sys.exit(1)
except (
ge_exceptions.ConfigNotFoundError,
ge_exceptions.InvalidConfigError,
) as err:
cli_message("<red>{}</red>".format(err.message))
sys.exit(1)
except ge_exceptions.PluginModuleNotFoundError as err:
cli_message(err.cli_colored_message)
sys.exit(1)
except ge_exceptions.PluginClassNotFoundError as err:
cli_message(err.cli_colored_message)
sys.exit(1)
except ge_exceptions.InvalidConfigurationYamlError as err:
cli_message(f"<red>{str(err)}</red>")
sys.exit(1)
def test_site_builder_with_custom_site_section_builders_config(
tmp_path_factory):
"""Test that site builder can handle partially specified custom site_section_builders config"""
base_dir = str(tmp_path_factory.mktemp("project_dir"))
project_dir = os.path.join(base_dir, "project_path")
os.mkdir(project_dir)
# fixture config swaps site section builder source stores and specifies custom run_id_filters
shutil.copy(
file_relative_path(
__file__,
"../test_fixtures/great_expectations_custom_local_site_config.yml"
), str(os.path.join(project_dir, "great_expectations.yml")))
context = DataContext(context_root_dir=project_dir)
local_site_config = context._project_config.data_docs_sites.get(
"local_site")
module_name = 'great_expectations.render.renderer.site_builder'
site_builder = instantiate_class_from_config(
config=local_site_config,
runtime_environment={
"data_context": context,
"root_directory": context.root_directory,
"site_name": 'local_site'
},
config_defaults={"module_name": module_name})
site_section_builders = site_builder.site_section_builders
expectations_site_section_builder = site_section_builders["expectations"]
assert isinstance(expectations_site_section_builder.source_store,
ValidationsStore)
validations_site_section_builder = site_section_builders["validations"]
assert isinstance(validations_site_section_builder.source_store,
ExpectationsStore)
assert validations_site_section_builder.run_id_filter == {
"ne": "custom_validations_filter"
}
profiling_site_section_builder = site_section_builders["profiling"]
assert isinstance(validations_site_section_builder.source_store,
ExpectationsStore)
assert profiling_site_section_builder.run_id_filter == {
"eq": "custom_profiling_filter"
}
def test_cli_datasorce_list(caplog, empty_data_context, filesystem_csv_2):
"""Test an empty project and after adding a single datasource."""
project_root_dir = empty_data_context.root_directory
context = DataContext(project_root_dir)
runner = CliRunner(mix_stderr=False)
result = runner.invoke(cli, ["datasource", "list", "-d", project_root_dir],
catch_exceptions=False)
stdout = result.output.strip()
assert "[]" in stdout
assert context.list_datasources() == []
context.add_datasource(
"wow_a_datasource",
module_name="great_expectations.datasource",
class_name="PandasDatasource",
generators={
"subdir_reader": {
"class_name": "SubdirReaderBatchKwargsGenerator",
"base_directory": str(filesystem_csv_2),
}
},
)
assert context.list_datasources() == [{
"name": "wow_a_datasource",
"class_name": "PandasDatasource"
}]
runner = CliRunner(mix_stderr=False)
result = runner.invoke(cli, ["datasource", "list", "-d", project_root_dir],
catch_exceptions=False)
stdout = result.output.strip()
if PY2:
# deal with legacy python dictionary sorting
assert ("'name': 'wow_a_datasource'"
and "'class_name': 'PandasDatasource'" in stdout)
assert len(stdout) >= 60 and len(stdout) <= 70
else:
assert (
"[{'name': 'wow_a_datasource', 'class_name': 'PandasDatasource'}]"
in stdout)
assert_no_logging_messages_or_tracebacks(caplog, result)
def suite_new(suite, directory, view, batch_kwargs):
"""
Create a new expectation suite.
Great Expectations will choose a couple of columns and generate expectations about them
to demonstrate some examples of assertions you can make about your data.
"""
try:
context = DataContext(directory)
except ge_exceptions.ConfigNotFoundError as err:
cli_message("<red>{}</red>".format(err.message))
return
except ge_exceptions.ZeroDotSevenConfigVersionError as err:
_offer_to_install_new_template(err, context.root_directory)
return
if batch_kwargs is not None:
batch_kwargs = json.loads(batch_kwargs)
datasource_name = None
generator_name = None
generator_asset = None
try:
success, suite_name = create_expectation_suite_impl(
context,
datasource_name=datasource_name,
generator_name=generator_name,
generator_asset=generator_asset,
batch_kwargs=batch_kwargs,
expectation_suite_name=suite,
additional_batch_kwargs={"limit": 1000},
show_intro_message=False,
open_docs=view,
)
if success:
cli_message("A new Expectation suite '{}' was added to your project".format(suite_name))
except (
ge_exceptions.DataContextError,
ge_exceptions.ProfilerError,
IOError,
SQLAlchemyError
) as e:
cli_message("<red>{}</red>".format(e))
sys.exit(1)
def test_suite_edit_with_invalid_json_batch_kwargs_raises_helpful_error(
caplog, empty_data_context
):
project_dir = empty_data_context.root_directory
context = DataContext(project_dir)
context.create_expectation_suite("foo")
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli,
["suite", "edit", "foo", "-d", project_dir, "--batch-kwargs", "'{foobar}'"],
catch_exceptions=False,
)
stdout = result.output
assert result.exit_code == 1
assert "Please check that your batch_kwargs are valid JSON." in stdout
# assert "Expecting value" in stdout # The exact message in the exception varies in Py 2, Py 3
assert_no_logging_messages_or_tracebacks(caplog, result)
def test_suite_edit_with_non_existent_datasource_shows_helpful_error_message(
caplog, empty_data_context):
project_dir = empty_data_context.root_directory
context = DataContext(project_dir)
context.create_expectation_suite("foo")
assert context.list_expectation_suites()[0].expectation_suite_name == "foo"
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli,
"suite edit foo -d {} --datasource not_real".format(project_dir),
catch_exceptions=False,
)
assert result.exit_code == 1
assert (
"Unable to load datasource `not_real` -- no configuration found or invalid configuration."
in result.output)
assert_no_logging_messages_or_tracebacks(caplog, result)
def test_store_list_with_zero_stores(caplog, empty_data_context):
project_dir = empty_data_context.root_directory
context = DataContext(project_dir)
context._project_config.stores = {}
context._save_project_config()
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli,
"store list -d {}".format(project_dir),
catch_exceptions=False,
)
assert result.exit_code == 1
assert (
"Your configuration file is not a valid yml file likely due to a yml syntax error"
in result.output.strip())
assert_no_logging_messages_or_tracebacks(caplog, result)
def test_checkpoint_new_works_if_checkpoints_directory_is_missing(
mock_emit, caplog, titanic_data_context_stats_enabled, titanic_expectation_suite
):
context = titanic_data_context_stats_enabled
root_dir = context.root_directory
checkpoints_dir = os.path.join(root_dir, context.CHECKPOINTS_DIR)
shutil.rmtree(checkpoints_dir)
assert not os.path.isdir(checkpoints_dir)
assert context.list_checkpoints() == []
context.save_expectation_suite(titanic_expectation_suite)
assert context.list_expectation_suite_names() == ["Titanic.warning"]
mock_emit.reset_mock()
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli,
f"checkpoint new passengers Titanic.warning -d {root_dir}",
input="1\n1\n",
catch_exceptions=False,
)
stdout = result.stdout
assert result.exit_code == 0
assert "A checkpoint named `passengers` was added to your project" in stdout
assert mock_emit.call_count == 2
assert mock_emit.call_args_list == [
mock.call(
{"event_payload": {}, "event": "data_context.__init__", "success": True}
),
mock.call(
{"event": "cli.checkpoint.new", "event_payload": {}, "success": True}
),
]
expected_checkpoint = os.path.join(
root_dir, context.CHECKPOINTS_DIR, "passengers.yml"
)
assert os.path.isfile(expected_checkpoint)
# Newup a context for additional assertions
context = DataContext(root_dir)
assert context.list_checkpoints() == ["passengers"]
assert_no_logging_messages_or_tracebacks(caplog, result)
def test_suite_edit_on_exsiting_suite_one_datasources_with_datasource_arg_and_batch_kwargs(
caplog, titanic_data_context,
):
"""
Given:
- the suite foo exists
- the a datasource bar exists
- and the users runs this
great_expectations suite edit foo --datasource bar --batch-kwargs '{"path": "data/10k.csv"}'
Then:
- The user gets a working notebook
"""
project_dir = titanic_data_context.root_directory
context = DataContext(project_dir)
context.create_expectation_suite("foo")
runner = CliRunner(mix_stderr=False)
batch_kwargs = {"path": os.path.join(project_dir, "../", "data", "Titanic.csv")}
result = runner.invoke(
cli,
[
"suite",
"edit",
"foo",
"-d",
project_dir,
"--batch-kwargs",
json.dumps(batch_kwargs),
"--datasource",
"mydatasource",
"--no-jupyter",
],
catch_exceptions=False,
)
stdout = result.output
assert result.exit_code == 0
assert "To continue editing this suite, run" in stdout
expected_notebook_path = os.path.join(project_dir, "uncommitted", "foo.ipynb")
assert os.path.isfile(expected_notebook_path)
expected_suite_path = os.path.join(project_dir, "expectations", "foo.json")
assert os.path.isfile(expected_suite_path)
assert_no_logging_messages_or_tracebacks(caplog, result)
def test_suite_edit_on_exsiting_suite_one_datasources_with_batch_kwargs_without_datasource_raises_helpful_error(
mock_webbrowser, mock_subprocess, caplog, titanic_data_context,
):
"""
Given:
- the suite foo exists
- the a datasource exists
- and the users runs this
great_expectations suite edit foo --batch-kwargs '{"path": "data/10k.csv"}'
Then:
- The user should see a nice error and the program halts before notebook
compilation.
- NOT open Data Docs
- NOT open jupyter
'"""
project_dir = titanic_data_context.root_directory
context = DataContext(project_dir)
context.create_expectation_suite("foo")
runner = CliRunner(mix_stderr=False)
batch_kwargs = {"path": "../data/Titanic.csv"}
result = runner.invoke(
cli,
[
"suite",
"edit",
"foo",
"-d",
project_dir,
"--batch-kwargs",
json.dumps(batch_kwargs),
],
catch_exceptions=False,
)
stdout = result.output
assert result.exit_code == 1
assert "Please check that your batch_kwargs are able to load a batch." in stdout
assert "Unable to load datasource `None`" in stdout
assert mock_webbrowser.call_count == 0
assert mock_subprocess.call_count == 0
assert_no_logging_messages_or_tracebacks(caplog, result)
def _run_cli_datasource_new_path_test(
context: DataContext, args: str, invocation_input: str, base_path: str
) -> None:
root_dir = context.root_directory
runner = CliRunner(mix_stderr=True)
result = runner.invoke(
cli,
args=args,
input=invocation_input,
catch_exceptions=False,
)
assert result.exit_code == 0
_run_notebook(context)
# Renew a context since we executed a notebook in a different process
del context
context = DataContext(root_dir)
assert context.list_datasources() == [
{
"name": "my_datasource",
"class_name": "Datasource",
"module_name": "great_expectations.datasource",
"execution_engine": {
"module_name": "great_expectations.execution_engine",
"class_name": "PandasExecutionEngine",
},
"data_connectors": {
"default_inferred_data_connector_name": {
"default_regex": {
"group_names": ["data_asset_name"],
"pattern": "(.*)",
},
"module_name": "great_expectations.datasource.data_connector",
"base_directory": f"../../{base_path}",
"class_name": "InferredAssetFilesystemDataConnector",
},
"default_runtime_data_connector_name": {
"class_name": "RuntimeDataConnector",
"module_name": "great_expectations.datasource.data_connector",
"batch_identifiers": ["default_identifier_name"],
},
},
}
]
def test_suite_delete_with_one_suite(mock_emit, caplog, monkeypatch,
empty_data_context_stats_enabled):
project_dir = empty_data_context_stats_enabled.root_directory
context = DataContext(project_dir)
suite = context.create_expectation_suite("a.warning")
context.save_expectation_suite(suite)
mock_emit.reset_mock()
suite_dir = os.path.join(project_dir, "expectations", "a")
suite_path = os.path.join(suite_dir, "warning.json")
assert os.path.isfile(suite_path)
runner = CliRunner(mix_stderr=False)
monkeypatch.chdir(os.path.dirname(context.root_directory))
result = runner.invoke(
cli,
f"--v3-api suite delete a.warning",
catch_exceptions=False,
)
assert result.exit_code == 0
assert "Deleted the expectation suite named: a.warning" in result.output
# assert not os.path.isdir(suite_dir)
assert not os.path.isfile(suite_path)
assert mock_emit.call_count == 2
assert mock_emit.call_args_list == [
mock.call({
"event_payload": {},
"event": "data_context.__init__",
"success": True
}),
mock.call({
"event": "cli.suite.delete",
"event_payload": {},
"success": True
}),
]
assert_no_logging_messages_or_tracebacks(
my_caplog=caplog,
click_result=result,
)
def check_if_datasource_name_exists(context: DataContext,
datasource_name: str) -> bool:
"""
Check if a Datasource name already exists in the on-disk version of the given DataContext and if so raise an error
Args:
context: DataContext to check for existing Datasource
datasource_name: name of the proposed Datasource
Returns:
boolean True if datasource name exists in on-disk config, else False
"""
# TODO: 20210324 Anthony: Note reading the context from disk is a temporary fix to allow use in a notebook
# after test_yaml_config(). test_yaml_config() should update a copy of the in-memory data context rather than
# making changes directly to the in-memory context.
context_on_disk: DataContext = DataContext(context.root_directory)
return datasource_name in [
d["name"] for d in context_on_disk.list_datasources()
]
def load_data_context_with_error_handling(directory: str) -> DataContext:
"""Return a DataContext with good error handling and exit codes."""
# TODO consolidate all the myriad CLI tests into this
try:
context = DataContext(directory)
return context
except (
ge_exceptions.ConfigNotFoundError,
ge_exceptions.InvalidConfigError,
ge_exceptions.UnsupportedConfigVersionError,
) as err:
cli_message("<red>{}</red>".format(err.message))
sys.exit(1)
except ge_exceptions.PluginModuleNotFoundError as err:
cli_message(err.cli_colored_message)
sys.exit(1)
except ge_exceptions.PluginClassNotFoundError as err:
cli_message(err.cli_colored_message)
sys.exit(1)
def test_docs_build_no_view(
mock_webbrowser,
caplog,
site_builder_data_context_v013_with_html_store_titanic_random,
):
root_dir = (
site_builder_data_context_v013_with_html_store_titanic_random.root_directory
)
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli,
["docs", "build", "--no-view", "-d", root_dir],
input="\n",
catch_exceptions=False,
)
stdout = result.stdout
assert result.exit_code == 0
assert mock_webbrowser.call_count == 0
assert "Building" in stdout
assert "The following Data Docs sites will be built:" in stdout
assert "great_expectations/uncommitted/data_docs/local_site/index.html" in stdout
context = DataContext(root_dir)
obs_urls = context.get_docs_sites_urls()
assert len(obs_urls) == 2
assert (
"great_expectations/uncommitted/data_docs/local_site/index.html"
in obs_urls[0]["site_url"]
)
local_site_dir = os.path.join(root_dir, "uncommitted/data_docs/local_site/")
assert os.path.isdir(os.path.join(local_site_dir))
assert os.path.isfile(os.path.join(local_site_dir, "index.html"))
assert os.path.isdir(os.path.join(local_site_dir, "expectations"))
assert os.path.isdir(os.path.join(local_site_dir, "validations"))
assert_no_logging_messages_or_tracebacks(
my_caplog=caplog,
click_result=result,
allowed_deprecation_message=VALIDATION_OPERATORS_DEPRECATION_MESSAGE,
)
def initialized_sqlite_project(
mock_webbrowser, caplog, tmp_path_factory, titanic_sqlite_db_file
):
"""This is an initialized project through the CLI."""
project_dir = str(tmp_path_factory.mktemp("my_rad_project"))
engine = create_engine("sqlite:///{}".format(titanic_sqlite_db_file))
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli,
["init", "-d", project_dir],
input="\n\n2\n6\ntitanic\n{}\n\n\n1\nwarning\n\n\n\n".format(engine.url),
catch_exceptions=False,
)
assert result.exit_code == 0
assert mock_webbrowser.call_count == 1
assert (
"{}/great_expectations/uncommitted/data_docs/local_site/validations/warning/".format(
project_dir
)
in mock_webbrowser.call_args[0][0]
)
assert_no_logging_messages_or_tracebacks(caplog, result)
context = DataContext(os.path.join(project_dir, DataContext.GE_DIR))
assert isinstance(context, DataContext)
assert len(context.list_datasources()) == 1
assert context.list_datasources() == [
{
"class_name": "SqlAlchemyDatasource",
"name": "titanic",
"module_name": "great_expectations.datasource",
"credentials": {"url": str(engine.url)},
"data_asset_type": {
"class_name": "SqlAlchemyDataset",
"module_name": "great_expectations.dataset",
},
}
]
return project_dir
def test_cli_datasorce_new_connection_string(empty_data_context,
empty_sqlite_db, caplog,
monkeypatch):
project_root_dir = empty_data_context.root_directory
context = DataContext(project_root_dir)
assert context.list_datasources() == []
runner = CliRunner(mix_stderr=False)
monkeypatch.chdir(os.path.dirname(context.root_directory))
result = runner.invoke(
cli,
[
"--v3-api",
"datasource",
"new",
],
input=f"2\n6\nmynewsource\n{empty_sqlite_db.url}\n\n",
catch_exceptions=False,
)
stdout = result.stdout
assert "What data would you like Great Expectations to connect to?" in stdout
assert "Give your new Datasource a short name." in stdout
assert (
"Next, we will configure database credentials and store them in the `mynewsource` section"
in stdout)
assert "What is the url/connection string for the sqlalchemy connection?" in stdout
assert "Attempting to connect to your database. This may take a moment" in stdout
assert "Great Expectations connected to your database" in stdout
assert "A new datasource 'mynewsource' was added to your project." in stdout
assert result.exit_code == 0
config_path = os.path.join(project_root_dir, DataContext.GE_YML)
config = yaml.load(open(config_path))
datasources = config["datasources"]
assert "mynewsource" in datasources.keys()
data_source_class = datasources["mynewsource"]["data_asset_type"][
"class_name"]
assert data_source_class == "SqlAlchemyDataset"
assert_no_logging_messages_or_tracebacks(caplog, result)
def test_suite_edit_with_batch_kwargs_unable_to_load_a_batch_raises_helpful_error(
mock_webbrowser, mock_subprocess, caplog, monkeypatch,
empty_data_context):
"""
The command should:
- exit with a clear error message
- NOT open Data Docs
- NOT open jupyter
"""
project_dir = empty_data_context.root_directory
context = DataContext(project_dir)
context.create_expectation_suite("foo")
context.add_datasource("source", class_name="PandasDatasource")
runner = CliRunner(mix_stderr=False)
batch_kwargs = '{"table": "fake", "datasource": "source"}'
monkeypatch.chdir(os.path.dirname(context.root_directory))
result = runner.invoke(
cli,
[
"--v3-api",
"suite",
"edit",
"foo",
"--batch-kwargs",
batch_kwargs,
],
catch_exceptions=False,
)
stdout = result.output
assert result.exit_code == 1
assert "To continue editing this suite" not in stdout
assert "Please check that your batch_kwargs are able to load a batch." in stdout
assert mock_webbrowser.call_count == 0
assert mock_subprocess.call_count == 0
assert_no_logging_messages_or_tracebacks(
my_caplog=caplog,
click_result=result,
)
def test_suite_list_with_multiple_suites(caplog, empty_data_context):
project_dir = empty_data_context.root_directory
context = DataContext(project_dir)
context.create_expectation_suite("a.warning")
context.create_expectation_suite("b.warning")
context.create_expectation_suite("c.warning")
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli, "suite list -d {}".format(project_dir), catch_exceptions=False,
)
output = result.output
assert result.exit_code == 0
assert "3 Expectation Suites found:" in output
assert "a.warning" in output
assert "b.warning" in output
assert "c.warning" in output
assert_no_logging_messages_or_tracebacks(caplog, result)
def __init__(self, data_context=None, context_root_dir=None):
assert (
data_context or context_root_dir
), "Please provide a data_context object or a context_root_dir."
self.data_context = data_context or DataContext(
context_root_dir=context_root_dir)
self.upgrade_log = {
"skipped_upgrade": False,
"update_version": True,
"added_checkpoint_store": {},
}
self.upgrade_checklist = {
"stores": {},
"checkpoint_store_name": None,
}
self._generate_upgrade_checklist()
def docs_build(directory, site_name, view=True):
"""
Build Data Docs for a project.
:param directory:
:param site_name: name for the first-level keys in the "data_docs_sites" section of the configuration
:param view:
"""
try:
context = DataContext(directory)
build_docs(context, site_name=site_name, view=view)
except ge_exceptions.ConfigNotFoundError as err:
cli_message("<red>{}</red>".format(err.message))
sys.exit(1)
except ge_exceptions.PluginModuleNotFoundError as err:
cli_message(err.cli_colored_message)
sys.exit(1)
except ge_exceptions.PluginClassNotFoundError as err:
cli_message(err.cli_colored_message)
sys.exit(1)
def initialized_project(mock_webbrowser, tmp_path_factory):
"""This is an initialized project through the CLI."""
project_dir = str(tmp_path_factory.mktemp("my_rad_project"))
os.makedirs(os.path.join(project_dir, "data"))
data_path = os.path.join(project_dir, "data/Titanic.csv")
fixture_path = file_relative_path(__file__, "../test_sets/Titanic.csv")
shutil.copy(fixture_path, data_path)
runner = CliRunner(mix_stderr=False)
_ = runner.invoke(
cli,
["init", "-d", project_dir],
input="Y\n1\n1\n{}\n\n\n\n".format(data_path, catch_exceptions=False),
)
assert mock_webbrowser.call_count == 1
assert "{}/great_expectations/uncommitted/data_docs/local_site/validations/warning/".format(project_dir) in mock_webbrowser.call_args[0][0]
context = DataContext(os.path.join(project_dir, DataContext.GE_DIR))
assert isinstance(context, DataContext)
assert len(context.list_datasources()) == 1
return project_dir
def suite_list(directory):
"""Lists available Expectation Suites."""
try:
context = DataContext(directory)
except ge_exceptions.ConfigNotFoundError as err:
cli_message("<red>{}</red>".format(err.message))
return
try:
suite_names = [
" - <cyan>{}</cyan>".format(suite_name)
for suite_name in context.list_expectation_suite_names()
]
if len(suite_names) == 0:
cli_message("No Expectation Suites found")
send_usage_message(
data_context=context,
event="cli.suite.list",
success=True
)
return
if len(suite_names) == 1:
list_intro_string = "1 Expectation Suite found:"
if len(suite_names) > 1:
list_intro_string = "{} Expectation Suites found:".format(len(suite_names))
cli_message_list(suite_names, list_intro_string)
send_usage_message(
data_context=context,
event="cli.suite.list",
success=True
)
except Exception as e:
send_usage_message(
data_context=context,
event="cli.suite.list",
success=False
)
raise e
def test_init_on_existing_project_with_multiple_datasources_exist_do_nothing(
mock_webbrowser,
caplog,
monkeypatch,
initialized_sqlite_project,
titanic_sqlite_db,
empty_sqlite_db,
):
project_dir = initialized_sqlite_project
ge_dir = os.path.join(project_dir, DataContext.GE_DIR)
context = DataContext(ge_dir)
datasource_name = "wow_a_datasource"
context = _add_datasource_and_credentials_to_context(
context, datasource_name, empty_sqlite_db)
assert len(context.list_datasources()) == 2
runner = CliRunner(mix_stderr=False)
monkeypatch.chdir(project_dir)
with pytest.warns(
UserWarning,
match="Warning. An existing `great_expectations.yml` was found"):
result = runner.invoke(
cli,
["--v3-api", "init"],
input="n\n",
catch_exceptions=False,
)
stdout = result.stdout
assert result.exit_code == 0
assert mock_webbrowser.call_count == 0
assert "Error: invalid input" not in stdout
assert "Always know what to expect from your data" in stdout
assert "This looks like an existing project that" in stdout
assert "appears complete" in stdout
assert "Would you like to build & view this project's Data Docs" in stdout
assert_no_logging_messages_or_tracebacks(caplog, result)
def test_cli_datasource_list(empty_data_context, empty_sqlite_db, caplog, monkeypatch):
"""Test an empty project and after adding a single datasource."""
project_root_dir = empty_data_context.root_directory
context = DataContext(project_root_dir)
runner = CliRunner(mix_stderr=False)
monkeypatch.chdir(os.path.dirname(context.root_directory))
result = runner.invoke(
cli,
"--v3-api datasource list",
catch_exceptions=False,
)
stdout = result.stdout.strip()
assert "No Datasources found" in stdout
assert context.list_datasources() == []
datasource_name = "wow_a_datasource"
_add_datasource_and_credentials_to_context(
context, datasource_name, empty_sqlite_db
)
runner = CliRunner(mix_stderr=False)
monkeypatch.chdir(os.path.dirname(context.root_directory))
result = runner.invoke(
cli,
["--v3-api", "datasource", "list"],
catch_exceptions=False,
)
expected_output = """\
Using v3 (Batch Request) API\x1b[0m
1 Datasource found:[0m
[0m
- [36mname:[0m wow_a_datasource[0m
[36mclass_name:[0m SqlAlchemyDatasource[0m
""".strip()
stdout = result.stdout.strip()
assert stdout == expected_output
assert_no_logging_messages_or_tracebacks(caplog, result)
def test_validation_operator_list_with_zero_validation_operators(
caplog, empty_data_context):
project_dir = empty_data_context.root_directory
context = DataContext(project_dir)
context._project_config.validation_operators = {}
context._save_project_config()
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli,
f"validation-operator list -d {project_dir}",
catch_exceptions=False,
)
assert result.exit_code == 0
assert "No Validation Operators found" in result.output
assert_no_logging_messages_or_tracebacks(
my_caplog=caplog,
click_result=result,
allowed_deprecation_message=VALIDATION_OPERATORS_DEPRECATION_MESSAGE,
)
