def store_list(directory):
"""List known Stores."""
context = toolkit.load_data_context_with_error_handling(directory)
try:
stores = context.list_stores()
if len(stores) == 0:
cli_message("No Stores found")
toolkit.send_usage_message(data_context=context,
event="cli.store.list",
success=True)
return
elif len(stores) == 1:
list_intro_string = "1 Store found:"
else:
list_intro_string = "{} Stores found:".format(len(stores))
cli_message(list_intro_string)
for store in stores:
cli_message("")
cli_message_dict(store)
toolkit.send_usage_message(data_context=context,
event="cli.store.list",
success=True)
except Exception as e:
toolkit.send_usage_message(data_context=context,
event="cli.store.list",
success=False)
raise e
def suite_delete(suite, directory):
"""
Delete an expectation suite from the expectation store.
"""
usage_event = "cli.suite.delete"
context = toolkit.load_data_context_with_error_handling(directory)
suite_names = context.list_expectation_suite_names()
if not suite_names:
toolkit.exit_with_failure_message_and_stats(
context,
usage_event,
"</red>No expectation suites found in the project.</red>",
)
if suite not in suite_names:
toolkit.exit_with_failure_message_and_stats(
context, usage_event, f"No expectation suite named {suite} found."
)
context.delete_expectation_suite(suite)
cli_message(f"Deleted the expectation suite named: {suite}")
send_usage_message(
data_context=context,
event=usage_event,
api_version="v2",
success=True,
)
def checkpoint_list(directory) -> None:
"""List configured checkpoints. (Experimental)"""
context = toolkit.load_data_context_with_error_handling(directory)
checkpoints = context.list_checkpoints()
if not checkpoints:
cli_message(
"No checkpoints found.\n"
" - Use the command `great_expectations checkpoint new` to create one."
)
send_usage_message(
data_context=context,
event="cli.checkpoint.list",
api_version="v2",
success=True,
)
sys.exit(0)
number_found = len(checkpoints)
plural = "s" if number_found > 1 else ""
message = f"Found {number_found} checkpoint{plural}."
pretty_list = [f" - <cyan>{cp}</cyan>" for cp in checkpoints]
cli_message_list(pretty_list, list_intro_string=message)
send_usage_message(
data_context=context,
event="cli.checkpoint.list",
api_version="v2",
success=True,
)
def suite_list(directory):
"""Lists available Expectation Suites."""
context = toolkit.load_data_context_with_error_handling(directory)
try:
suite_names = [
f" - <cyan>{suite_name}</cyan>"
for suite_name in context.list_expectation_suite_names()
]
if len(suite_names) == 0:
cli_message("No Expectation Suites found")
toolkit.send_usage_message(data_context=context,
event="cli.suite.list",
success=True)
return
elif len(suite_names) == 1:
list_intro_string = "1 Expectation Suite found:"
else:
list_intro_string = f"{len(suite_names)} Expectation Suites found:"
cli_message_list(suite_names, list_intro_string)
toolkit.send_usage_message(data_context=context,
event="cli.suite.list",
success=True)
except Exception as e:
toolkit.send_usage_message(data_context=context,
event="cli.suite.list",
success=False)
raise e
def clean_data_docs(directory, site_name=None, all=None) -> None:
"""Delete data docs"""
context = toolkit.load_data_context_with_error_handling(directory)
failed = True
if site_name is None and all is None:
cli_message("<red>{}</red>".format(
"Please specify --all to remove all sites or specify a specific site using "
"--site_name"))
sys.exit(1)
context.clean_data_docs(site_name=site_name)
failed = False
if not failed and context is not None:
send_usage_message(
data_context=context,
event="cli.docs.clean",
api_version="v2",
success=True,
)
cli_message("<green>Cleaned data docs</green>")
if failed and context is not None:
send_usage_message(
data_context=context,
event="cli.docs.clean",
api_version="v2",
success=False,
)
def docs_list(directory) -> None:
"""List known Data Docs Sites."""
context = toolkit.load_data_context_with_error_handling(directory)
docs_sites_url_dicts = context.get_docs_sites_urls()
docs_sites_strings = [
" - <cyan>{}</cyan>: {}".format(
docs_site_dict["site_name"],
docs_site_dict.get("site_url") or
f"site configured but does not exist. Run the following command to build site: great_expectations "
f'docs build --site-name {docs_site_dict["site_name"]}',
) for docs_site_dict in docs_sites_url_dicts
]
if len(docs_sites_strings) == 0:
cli_message("No Data Docs sites found")
else:
list_intro_string = _build_intro_string(docs_sites_strings)
cli_message_list(docs_sites_strings, list_intro_string)
send_usage_message(
data_context=context,
event="cli.docs.list",
api_version="v2",
success=True,
)
def checkpoint_new(checkpoint, suite, directory, datasource):
"""Create a new checkpoint for easy deployments. (Experimental)"""
suite_name = suite
usage_event = "cli.checkpoint.new"
context = toolkit.load_data_context_with_error_handling(directory)
_verify_checkpoint_does_not_exist(context, checkpoint, usage_event)
suite: ExpectationSuite = toolkit.load_expectation_suite(
context, suite_name, usage_event
)
datasource = toolkit.select_datasource(context, datasource_name=datasource)
if datasource is None:
toolkit.send_usage_message(context, usage_event, success=False)
sys.exit(1)
_, _, _, batch_kwargs = toolkit.get_batch_kwargs(context, datasource.name)
_ = context.add_checkpoint(
name=checkpoint,
**{
"class_name": "LegacyCheckpoint",
"batches": [
{
"batch_kwargs": dict(batch_kwargs),
"expectation_suite_names": [suite.expectation_suite_name],
}
],
},
)
cli_message(
f"""<green>A Checkpoint named `{checkpoint}` was added to your project!</green>
- To run this Checkpoint, run `great_expectations checkpoint run {checkpoint}`"""
)
toolkit.send_usage_message(context, usage_event, success=True)
def docs_build(directory, site_name, view=True, assume_yes=False):
"""Build Data Docs for a project."""
context = toolkit.load_data_context_with_error_handling(directory)
build_docs(context, site_name=site_name, view=view, assume_yes=assume_yes)
toolkit.send_usage_message(data_context=context,
event="cli.docs.build",
success=True)
def checkpoint_script(checkpoint, directory):
"""
Create a python script to run a checkpoint. (Experimental)
Checkpoints can be run directly without this script using the
`great_expectations checkpoint run` command.
This script is provided for those who wish to run checkpoints via python.
"""
context = toolkit.load_data_context_with_error_handling(directory)
usage_event = "cli.checkpoint.script"
# Attempt to load the checkpoint and deal with errors
_ = toolkit.load_checkpoint(context, checkpoint, usage_event)
script_name = f"run_{checkpoint}.py"
script_path = os.path.join(
context.root_directory, context.GE_UNCOMMITTED_DIR, script_name
)
if os.path.isfile(script_path):
toolkit.exit_with_failure_message_and_stats(
context,
usage_event,
f"""<red>Warning! A script named {script_name} already exists and this command will not overwrite it.</red>
- Existing file path: {script_path}""",
)
_write_checkpoint_script_to_disk(context.root_directory, checkpoint, script_path)
cli_message(
f"""<green>A python script was created that runs the checkpoint named: `{checkpoint}`</green>
- The script is located in `great_expectations/uncommitted/run_{checkpoint}.py`
- The script can be run with `python great_expectations/uncommitted/run_{checkpoint}.py`"""
)
toolkit.send_usage_message(context, event=usage_event, success=True)
def checkpoint_run(checkpoint, directory):
"""Run a checkpoint. (Experimental)"""
usage_event = "cli.checkpoint.run"
context = toolkit.load_data_context_with_error_handling(
directory=directory, from_cli_upgrade_command=False
)
checkpoint: Checkpoint = toolkit.load_checkpoint(
context,
checkpoint,
usage_event,
)
try:
results = checkpoint.run()
except Exception as e:
toolkit.exit_with_failure_message_and_stats(
context, usage_event, f"<red>{e}</red>"
)
if not results["success"]:
cli_message("Validation failed!")
toolkit.send_usage_message(context, event=usage_event, success=True)
print_validation_operator_results_details(results)
sys.exit(1)
cli_message("Validation succeeded!")
toolkit.send_usage_message(context, event=usage_event, success=True)
print_validation_operator_results_details(results)
sys.exit(0)
def project_upgrade(directory):
"""Upgrade a project after installing the next Great Expectations major version."""
cli_message("\nChecking project...")
cli_message(SECTION_SEPARATOR)
if load_data_context_with_error_handling(directory=directory,
from_cli_upgrade_command=True):
up_to_date_message = (
"Your project is up-to-date - no further upgrade is necessary.\n")
cli_message(f"<green>{up_to_date_message}</green>")
sys.exit(0)
def _suite_scaffold(suite: str, directory: str, jupyter: bool) -> None:
usage_event = "cli.suite.scaffold"
suite_name = suite
context = toolkit.load_data_context_with_error_handling(directory)
notebook_filename = f"scaffold_{suite_name}.ipynb"
notebook_path = _get_notebook_path(context, notebook_filename)
if suite_name in context.list_expectation_suite_names():
toolkit.tell_user_suite_exists(suite_name)
if os.path.isfile(notebook_path):
cli_message(
f" - If you wish to adjust your scaffolding, you can open this notebook with jupyter: `{notebook_path}` <red>(Please note that if you run that notebook, you will overwrite your existing suite.)</red>"
)
send_usage_message(
data_context=context,
event=usage_event,
api_version="v2",
success=False,
)
sys.exit(1)
datasource = toolkit.select_datasource(context)
if datasource is None:
send_usage_message(
data_context=context,
event=usage_event,
api_version="v2",
success=False,
)
sys.exit(1)
_suite = context.create_expectation_suite(suite_name)
_, _, _, batch_kwargs = get_batch_kwargs(context,
datasource_name=datasource.name)
renderer = SuiteScaffoldNotebookRenderer(context, _suite, batch_kwargs)
renderer.render_to_disk(notebook_path)
send_usage_message(
data_context=context,
event=usage_event,
api_version="v2",
success=True,
)
if jupyter:
toolkit.launch_jupyter_notebook(notebook_path)
else:
cli_message(
f"To continue scaffolding this suite, run `jupyter notebook {notebook_path}`"
)
def checkpoint_new(checkpoint, suite, directory, datasource, legacy):
"""Create a new checkpoint for easy deployments. (Experimental)"""
if legacy:
suite_name = suite
usage_event = "cli.checkpoint.new"
context = toolkit.load_data_context_with_error_handling(directory)
ge_config_version = context.get_config().config_version
if ge_config_version >= 3:
cli_message(
f"""<red>The `checkpoint new` CLI command is not yet implemented for Great Expectations config versions >= 3.</red>"""
)
toolkit.send_usage_message(context, usage_event, success=False)
sys.exit(1)
_verify_checkpoint_does_not_exist(context, checkpoint, usage_event)
suite: ExpectationSuite = toolkit.load_expectation_suite(
context, suite_name, usage_event)
datasource = toolkit.select_datasource(context,
datasource_name=datasource)
if datasource is None:
toolkit.send_usage_message(context, usage_event, success=False)
sys.exit(1)
_, _, _, batch_kwargs = toolkit.get_batch_kwargs(
context, datasource.name)
_ = context.add_checkpoint(
name=checkpoint,
**{
"class_name":
"LegacyCheckpoint",
"validation_operator_name":
"action_list_operator",
"batches": [{
"batch_kwargs":
dict(batch_kwargs),
"expectation_suite_names": [suite.expectation_suite_name],
}],
},
)
cli_message(
f"""<green>A checkpoint named `{checkpoint}` was added to your project!</green>
- To run this checkpoint run `great_expectations checkpoint run {checkpoint}`"""
)
toolkit.send_usage_message(context, usage_event, success=True)
# TODO: <Rob>Rob</Rob> Add flow for new style checkpoints
else:
pass
def checkpoint_run(checkpoint, directory):
"""Run a checkpoint. (Experimental)"""
usage_event = "cli.checkpoint.run"
context = toolkit.load_data_context_with_error_handling(
directory=directory, from_cli_upgrade_command=False)
ge_config_version = context.get_config().config_version
if ge_config_version >= 3:
cli_message(
f"""<red>The `checkpoint run` CLI command is not yet implemented for Great Expectations config versions >= 3.</red>"""
)
toolkit.send_usage_message(context, usage_event, success=False)
sys.exit(1)
checkpoint: Checkpoint = toolkit.load_checkpoint(
context,
checkpoint,
usage_event,
)
try:
results = checkpoint.run()
except Exception as e:
toolkit.exit_with_failure_message_and_stats(context, usage_event,
f"<red>{e}</red>")
if not results["success"]:
cli_message("Validation failed!")
toolkit.send_usage_message(context, event=usage_event, success=True)
print_validation_operator_results_details(results)
sys.exit(1)
cli_message("Validation succeeded!")
toolkit.send_usage_message(context, event=usage_event, success=True)
print_validation_operator_results_details(results)
sys.exit(0)
def _suite_edit(
suite,
datasource,
directory,
jupyter,
batch_kwargs,
usage_event,
suppress_usage_message=False,
):
# suppress_usage_message flag is for the situation where _suite_edit is called by _suite_new().
# when called by _suite_new(), the flag will be set to False, otherwise it will default to True
batch_kwargs_json = batch_kwargs
batch_kwargs = None
context = toolkit.load_data_context_with_error_handling(directory)
try:
suite = toolkit.load_expectation_suite(context, suite, usage_event)
citations = suite.get_citations(require_batch_kwargs=True)
if batch_kwargs_json:
try:
batch_kwargs = json.loads(batch_kwargs_json)
if datasource:
batch_kwargs["datasource"] = datasource
_batch = toolkit.load_batch(context, suite, batch_kwargs)
except json_parse_exception as je:
cli_message(
"<red>Please check that your batch_kwargs are valid JSON.\n{}</red>".format(
je
)
)
if not suppress_usage_message:
send_usage_message(
data_context=context,
event=usage_event,
api_version="v2",
success=True,
)
sys.exit(1)
except ge_exceptions.DataContextError:
cli_message(
"<red>Please check that your batch_kwargs are able to load a batch.</red>"
)
if not suppress_usage_message:
send_usage_message(
data_context=context,
event=usage_event,
api_version="v2",
success=False,
)
sys.exit(1)
except ValueError as ve:
cli_message(
"<red>Please check that your batch_kwargs are able to load a batch.\n{}</red>".format(
ve
)
)
if not suppress_usage_message:
send_usage_message(
data_context=context,
event=usage_event,
api_version="v2",
success=False,
)
sys.exit(1)
elif citations:
citation = citations[-1]
batch_kwargs = citation.get("batch_kwargs")
if not batch_kwargs:
cli_message(
"""
A batch of data is required to edit the suite - let's help you to specify it."""
)
additional_batch_kwargs = None
try:
data_source = toolkit.select_datasource(
context, datasource_name=datasource
)
except ValueError as ve:
cli_message(f"<red>{ve}</red>")
send_usage_message(
data_context=context,
event=usage_event,
api_version="v2",
success=False,
)
sys.exit(1)
if not data_source:
cli_message("<red>No datasources found in the context.</red>")
if not suppress_usage_message:
send_usage_message(
data_context=context,
event=usage_event,
api_version="v2",
success=False,
)
sys.exit(1)
if batch_kwargs is None:
(
datasource_name,
batch_kwargs_generator,
data_asset,
batch_kwargs,
) = get_batch_kwargs(
context,
datasource_name=data_source.name,
batch_kwargs_generator_name=None,
data_asset_name=None,
additional_batch_kwargs=additional_batch_kwargs,
)
notebook_name = f"edit_{suite.expectation_suite_name}.ipynb"
notebook_path = _get_notebook_path(context, notebook_name)
SuiteEditNotebookRenderer.from_data_context(context).render_to_disk(
suite, notebook_path, batch_kwargs
)
if not jupyter:
cli_message(
f"To continue editing this suite, run <green>jupyter notebook {notebook_path}</green>"
)
payload = edit_expectation_suite_usage_statistics(
data_context=context, expectation_suite_name=suite.expectation_suite_name
)
if not suppress_usage_message:
send_usage_message(
data_context=context,
event=usage_event,
event_payload=payload,
api_version="v2",
success=True,
)
if jupyter:
toolkit.launch_jupyter_notebook(notebook_path)
except Exception as e:
send_usage_message(
data_context=context,
event=usage_event,
api_version="v2",
success=False,
)
raise e
def _suite_new(
suite: str,
directory: str,
empty: bool,
jupyter: bool,
view: bool,
batch_kwargs,
usage_event: str,
) -> None:
# TODO break this up into demo and new
context = toolkit.load_data_context_with_error_handling(directory)
datasource_name = None
generator_name = None
data_asset_name = None
try:
if batch_kwargs is not None:
batch_kwargs = json.loads(batch_kwargs)
success, suite_name, profiling_results = toolkit.create_expectation_suite(
context,
datasource_name=datasource_name,
batch_kwargs_generator_name=generator_name,
data_asset_name=data_asset_name,
batch_kwargs=batch_kwargs,
expectation_suite_name=suite,
additional_batch_kwargs={"limit": 1000},
empty_suite=empty,
show_intro_message=False,
open_docs=view,
)
if success:
if empty:
if jupyter:
cli_message(
"""<green>Because you requested an empty suite, we'll open a notebook for you now to edit it!
If you wish to avoid this you can add the `--no-jupyter` flag.</green>\n\n"""
)
send_usage_message(
data_context=context,
event=usage_event,
api_version="v2",
success=True,
)
_suite_edit(
suite_name,
datasource_name,
directory,
jupyter=jupyter,
batch_kwargs=batch_kwargs,
usage_event="cli.suite.edit", # or else we will be sending `cli.suite.new` which is incorrect
suppress_usage_message=True, # dont want actually send usage_message since the function call is not the result of actual usage
)
else:
send_usage_message(
data_context=context,
event=usage_event,
api_version="v2",
success=False,
)
except (
ge_exceptions.DataContextError,
ge_exceptions.ProfilerError,
OSError,
SQLAlchemyError,
) as e:
cli_message(f"<red>{e}</red>")
send_usage_message(
data_context=context,
event=usage_event,
api_version="v2",
success=False,
)
sys.exit(1)
except Exception as e:
send_usage_message(
data_context=context,
event=usage_event,
api_version="v2",
success=False,
)
raise e
