def checkpoint(ctx):
"""
Checkpoint operations
A Checkpoint is a bundle of one or more batches of data with one or more
Expectation Suites.
A Checkpoint can be as simple as one batch of data paired with one
Expectation Suite.
A Checkpoint can be as complex as many batches of data across different
datasources paired with one or more Expectation Suites each.
"""
directory: str = toolkit.parse_cli_config_file_location(
config_file_location=ctx.obj.config_file_location).get("directory")
context: DataContext = toolkit.load_data_context_with_error_handling(
directory=directory,
from_cli_upgrade_command=False,
)
# TODO consider moving this all the way up in to the CLIState constructor
ctx.obj.data_context = context
usage_stats_prefix = f"cli.checkpoint.{ctx.invoked_subcommand}"
toolkit.send_usage_message(
data_context=context,
event=f"{usage_stats_prefix}.begin",
success=True,
)
ctx.obj.usage_event_end = f"{usage_stats_prefix}.end"
def docs_list(ctx):
"""List known Data Docs sites."""
context = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
docs_sites_url_dicts = context.get_docs_sites_urls()
try:
if len(docs_sites_url_dicts) == 0:
cli_message("No Data Docs sites found")
else:
docs_sites_strings = [
" - <cyan>{}</cyan>: {}".format(
docs_site_dict["site_name"],
docs_site_dict.get("site_url") or
f"site configured but does not exist. Run the following command to build site: great_expectations "
f'docs build --site-name {docs_site_dict["site_name"]}',
) for docs_site_dict in docs_sites_url_dicts
]
list_intro_string = _build_intro_string(docs_sites_strings)
cli_message_list(docs_sites_strings, list_intro_string)
toolkit.send_usage_message(data_context=context,
event=usage_event_end,
success=True)
except Exception as e:
toolkit.exit_with_failure_message_and_stats(
context=context,
usage_event=usage_event_end,
message=f"<red>{e}</red>",
)
return
def datasource_list(ctx):
"""List known Datasources."""
context = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
try:
datasources = context.list_datasources()
cli_message(_build_datasource_intro_string(datasources))
for datasource in datasources:
cli_message("")
cli_message_dict(
{
"name": datasource["name"],
"class_name": datasource["class_name"],
}
)
toolkit.send_usage_message(
data_context=context, event=usage_event_end, success=True
)
except Exception as e:
toolkit.exit_with_failure_message_and_stats(
context=context,
usage_event=usage_event_end,
message=f"<red>{e}</red>",
)
return
def checkpoint_run(ctx, checkpoint):
"""Run a Checkpoint."""
usage_event: str = "cli.checkpoint.run"
context: DataContext = ctx.obj.data_context
try:
result: CheckpointResult = toolkit.run_checkpoint(
context=context,
checkpoint_name=checkpoint,
usage_event=usage_event,
)
except Exception as e:
toolkit.exit_with_failure_message_and_stats(
context=context,
usage_event=usage_event,
message=f"<red>{e}</red>",
)
return
if not result["success"]:
cli_message(string="Validation failed!")
toolkit.send_usage_message(context, event=usage_event, success=True)
print_validation_operator_results_details(result=result)
sys.exit(1)
cli_message("Validation succeeded!")
toolkit.send_usage_message(context, event=usage_event, success=True)
print_validation_operator_results_details(result=result)
sys.exit(0)
def suite_new(ctx, expectation_suite, interactive, profile, batch_request, no_jupyter):
"""
Create a new empty Expectation Suite.
Edit in jupyter notebooks, or skip with the --no-jupyter flag.
"""
context: DataContext = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
error_message: Optional[str] = None
if not interactive and (profile or (batch_request is not None)):
error_message = """Using --profile flag and/or --batch-request <path to JSON file> option requires \
--interactive flag.
"""
if error_message is not None:
cli_message(string=f"<red>{error_message}</red>")
toolkit.send_usage_message(
data_context=context, event=usage_event_end, success=False
)
sys.exit(1)
_suite_new_workflow(
context=context,
expectation_suite_name=expectation_suite,
interactive=interactive,
profile=profile,
no_jupyter=no_jupyter,
usage_event=usage_event_end,
batch_request=batch_request,
)
def docs_build(ctx, site_name, view=True):
"""Build Data Docs for a project."""
context: DataContext = ctx.obj.data_context
build_docs(context, site_name=site_name, view=view, assume_yes=ctx.obj.assume_yes)
toolkit.send_usage_message(
data_context=context, event="cli.docs.build", success=True
)
def clean_data_docs(ctx, site_name=None, all=None):
"""Delete data docs"""
display_not_implemented_message_and_exit()
context = ctx.obj.data_context
failed = True
if site_name is None and all is None:
cli_message(
"<red>{}</red>".format(
"Please specify --all to remove all sites or specify a specific site using "
"--site_name"
)
)
sys.exit(1)
context.clean_data_docs(site_name=site_name)
failed = False
if not failed and context is not None:
toolkit.send_usage_message(
data_context=context, event="cli.docs.clean", success=True
)
cli_message("<green>{}</green>".format("Cleaned data docs"))
if failed and context is not None:
toolkit.send_usage_message(
data_context=context, event="cli.docs.clean", success=False
)
def suite_delete(ctx, suite):
"""
Delete an expectation suite from the expectation store.
"""
display_not_implemented_message_and_exit()
usage_event = "cli.suite.delete"
directory = toolkit.parse_cli_config_file_location(
config_file_location=ctx.obj.config_file_location).get("directory")
context = toolkit.load_data_context_with_error_handling(directory)
suite_names = context.list_expectation_suite_names()
if not suite_names:
toolkit.exit_with_failure_message_and_stats(
context,
usage_event,
"</red>No expectation suites found in the project.</red>",
)
if suite not in suite_names:
toolkit.exit_with_failure_message_and_stats(
context, usage_event, f"No expectation suite named {suite} found.")
context.delete_expectation_suite(suite)
cli_message(f"Deleted the expectation suite named: {suite}")
toolkit.send_usage_message(data_context=context,
event=usage_event,
success=True)
def datasource_new(ctx, name, jupyter):
"""Add a new Datasource to the data context."""
context = ctx.obj.data_context
toolkit.send_usage_message(data_context=context,
event="cli.datasource.new",
success=True)
_datasource_new_flow(context, datasource_name=name, jupyter=jupyter)
def docs_clean(ctx, site_name=None, all_sites=False):
"""
Remove all files from a Data Docs site.
This is a useful first step if you wish to completely re-build a site from scratch.
"""
context = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
if (site_name is None and all_sites is False) or (site_name and all_sites):
toolkit.exit_with_failure_message_and_stats(
data_context=context,
usage_event=usage_event_end,
message=
"<red>Please specify either --all to clean all sites or a specific site using --site-name</red>",
)
try:
# if site_name is None, context.clean_data_docs(site_name=site_name)
# will clean all sites.
context.clean_data_docs(site_name=site_name)
toolkit.send_usage_message(data_context=context,
event=usage_event_end,
success=True)
cli_message("<green>{}</green>".format("Cleaned data docs"))
except DataContextError as de:
toolkit.exit_with_failure_message_and_stats(
data_context=context,
usage_event=usage_event_end,
message=f"<red>{de}</red>",
)
def docs_build(ctx, site_name=None, no_view=False):
"""Build Data Docs for a project."""
context: DataContext = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
if site_name is not None and site_name not in context.get_site_names():
toolkit.exit_with_failure_message_and_stats(
data_context=context,
usage_event=usage_event_end,
message=
f"<red>The specified site name `{site_name}` does not exist in this project.</red>",
)
if site_name is None:
sites_to_build = context.get_site_names()
else:
sites_to_build = [site_name]
build_docs(
context,
usage_stats_event=usage_event_end,
site_names=sites_to_build,
view=not no_view,
assume_yes=ctx.obj.assume_yes,
)
toolkit.send_usage_message(data_context=context,
event=usage_event_end,
success=True)
def clean_data_docs(ctx, site_name=None, all_sites=False):
"""Delete data docs"""
context = ctx.obj.data_context
if (site_name is None and all_sites is False) or (site_name and all_sites):
toolkit.exit_with_failure_message_and_stats(
context,
usage_event="cli.docs.clean",
message=
"<red>Please specify either --all to remove all sites or a specific site using --site_name</red>",
)
try:
# if site_name is None, context.clean_data_docs(site_name=site_name)
# will clean all sites.
context.clean_data_docs(site_name=site_name)
toolkit.send_usage_message(data_context=context,
event="cli.docs.clean",
success=True)
cli_message("<green>{}</green>".format("Cleaned data docs"))
except DataContextError as de:
toolkit.exit_with_failure_message_and_stats(
context,
usage_event="cli.docs.clean",
message=f"<red>{de}</red>",
)
def delete_datasource(ctx, datasource):
"""Delete the datasource specified as an argument"""
context: DataContext = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
if not ctx.obj.assume_yes:
toolkit.confirm_proceed_or_exit(
confirm_prompt=f"""\nAre you sure you want to delete the Datasource "{datasource}" (this action is irreversible)?" """,
continuation_message=f"Datasource `{datasource}` was not deleted.",
exit_on_no=True,
data_context=context,
usage_stats_event=usage_event_end,
)
try:
context.delete_datasource(datasource)
except ValueError:
cli_message(f"<red>Datasource {datasource} could not be found.</red>")
toolkit.send_usage_message(context, event=usage_event_end, success=False)
sys.exit(1)
try:
context.get_datasource(datasource)
except ValueError:
cli_message("<green>{}</green>".format("Datasource deleted successfully."))
toolkit.send_usage_message(context, event=usage_event_end, success=True)
sys.exit(0)
def checkpoint_delete(ctx, checkpoint):
"""Delete a Checkpoint."""
usage_event: str = "cli.checkpoint.delete"
context: DataContext = ctx.obj.data_context
try:
toolkit.delete_checkpoint(
context=context,
checkpoint_name=checkpoint,
usage_event=usage_event,
assume_yes=ctx.obj.assume_yes,
)
toolkit.send_usage_message(context,
event="cli.checkpoint.delete",
success=True)
except Exception as e:
toolkit.exit_with_failure_message_and_stats(
context=context,
usage_event=usage_event,
message=f"<red>{e}</red>",
)
return
cli_message(f'Checkpoint "{checkpoint}" deleted.')
sys.exit(0)
def suite_edit(
ctx, expectation_suite, interactive, datasource_name, batch_request, no_jupyter
):
"""
Generate a Jupyter notebook for editing an existing Expectation Suite.
The SUITE argument is required. This is the name you gave to the suite
when you created it.
The edit command will help you specify a batch interactively. Or you can
specify them manually by providing --batch-request in valid JSON format.
Read more about specifying batches of data in the documentation: https://docs.greatexpectations.io/
"""
context: DataContext = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
error_message: Optional[str] = None
if not interactive and (
(datasource_name is not None) or (batch_request is not None)
):
error_message = """Using --datasource-name DATASOURCE_NAME option or --batch-request <path to JSON file> \
option requires --interactive flag.
"""
if interactive and (datasource_name is not None) and (batch_request is not None):
error_message = """Only one of --datasource-name DATASOURCE_NAME and --batch-request <path to JSON file> \
options can be used.
"""
if error_message is not None:
cli_message(string=f"<red>{error_message}</red>")
toolkit.send_usage_message(
data_context=context, event=usage_event_end, success=False
)
sys.exit(1)
additional_batch_request_args: Optional[
Dict[str, Union[str, int, Dict[str, Any]]]
] = {"limit": 1000}
_suite_edit_workflow(
context=context,
expectation_suite_name=expectation_suite,
profile=False,
usage_event=usage_event_end,
interactive=interactive,
no_jupyter=no_jupyter,
create_if_not_exist=False,
datasource_name=datasource_name,
batch_request=batch_request,
additional_batch_request_args=additional_batch_request_args,
suppress_usage_message=False,
assume_yes=False,
)
def send_backend_choice_usage_message(self, context: DataContext) -> None:
toolkit.send_usage_message(
data_context=context,
event="cli.new_ds_choice",
event_payload={
"type": self.datasource_type.value,
**self.usage_stats_payload,
},
success=True,
)
def suite_demo(ctx):
"""This command is not supported in the v3 (Batch Request) API."""
context = ctx.obj.data_context
usage_event = "cli.suite.demo"
toolkit.send_usage_message(data_context=context,
event=usage_event,
success=True)
cli_message(
"This command is not supported in the v3 (Batch Request) API. Please use `suite new` instead."
)
def suite_demo(ctx):
"""This command is not supported in the v3 (Batch Request) API."""
context: DataContext = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
toolkit.send_usage_message(
data_context=context, event=usage_event_end, success=True
)
cli_message(
string="This command is not supported in the v3 (Batch Request) API. Please use `suite new` instead."
)
def store_list(ctx):
"""List active Stores."""
context = ctx.obj.data_context
stores = context.list_active_stores()
cli_message(f"{len(stores)} active Stores found:")
for store in stores:
cli_message("")
cli_message_dict(store)
toolkit.send_usage_message(data_context=context,
event="cli.store.list",
success=True)
def _datasource_new_flow(
context: DataContext,
usage_event_end: str,
datasource_name: Optional[str] = None,
jupyter: bool = True,
) -> None:
files_or_sql_selection = click.prompt(
"""
What data would you like Great Expectations to connect to?
1. Files on a filesystem (for processing with Pandas or Spark)
2. Relational database (SQL)
""",
type=click.Choice(["1", "2"]),
show_choices=False,
)
if files_or_sql_selection == "1":
selected_files_backend = _prompt_for_execution_engine()
helper = _get_files_helper(
selected_files_backend,
context_root_dir=context.root_directory,
datasource_name=datasource_name,
)
elif files_or_sql_selection == "2":
if not _verify_sqlalchemy_dependent_modules():
return None
selected_database = _prompt_user_for_database_backend()
helper = _get_sql_yaml_helper_class(selected_database, datasource_name)
helper.send_backend_choice_usage_message(context)
if not helper.verify_libraries_installed():
return None
helper.prompt()
notebook_path = helper.create_notebook(context)
if jupyter is False:
cli_message(
f"To continue editing this Datasource, run <green>jupyter notebook {notebook_path}</green>"
)
toolkit.send_usage_message(context,
event=usage_event_end,
success=True)
return None
if notebook_path:
cli_message(
"""<green>Because you requested to create a new Datasource, we'll open a notebook for you now to complete it!</green>\n\n"""
)
toolkit.send_usage_message(context,
event=usage_event_end,
success=True)
toolkit.launch_jupyter_notebook(notebook_path)
def datasource_list(ctx):
"""List known Datasources."""
context = ctx.obj.data_context
datasources = context.list_datasources()
cli_message(_build_datasource_intro_string(datasources))
for datasource in datasources:
cli_message("")
cli_message_dict({
"name": datasource["name"],
"class_name": datasource["class_name"],
})
toolkit.send_usage_message(data_context=context,
event="cli.datasource.list",
success=True)
def project_check_config(ctx):
"""Check a config for validity and help with migrations."""
cli_message("Checking your config files for validity...\n")
directory = toolkit.parse_cli_config_file_location(
config_file_location=ctx.obj.config_file_location).get("directory")
is_config_ok, error_message, context = do_config_check(directory)
if context:
toolkit.send_usage_message(data_context=context,
event="cli.project.check_config",
success=True)
if not is_config_ok:
cli_message("Unfortunately, your config appears to be invalid:\n")
cli_message(f"<red>{error_message}</red>")
sys.exit(1)
cli_message("<green>Your config file appears valid!</green>")
def datasource(ctx):
"""Datasource operations"""
directory: str = toolkit.parse_cli_config_file_location(
config_file_location=ctx.obj.config_file_location).get("directory")
context: DataContext = toolkit.load_data_context_with_error_handling(
directory=directory,
from_cli_upgrade_command=False,
)
# TODO consider moving this all the way up in to the CLIState constructor
ctx.obj.data_context = context
usage_stats_prefix = f"cli.datasource.{ctx.invoked_subcommand}"
toolkit.send_usage_message(
data_context=context,
event=f"{usage_stats_prefix}.begin",
success=True,
)
ctx.obj.usage_event_end = f"{usage_stats_prefix}.end"
def datasource_new(ctx):
"""Add a new datasource to the data context."""
display_not_implemented_message_and_exit()
context = ctx.obj.data_context
datasource_name, data_source_type = add_datasource(context)
if datasource_name:
cli_message(
"A new datasource '{}' was added to your project.".format(datasource_name)
)
toolkit.send_usage_message(
data_context=context, event="cli.datasource.new", success=True
)
else: # no datasource was created
toolkit.send_usage_message(
data_context=context, event="cli.datasource.new", success=False
)
sys.exit(1)
def suite_delete(ctx, suite):
"""
Delete an expectation suite from the expectation store.
"""
context: DataContext = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
try:
suite_names: List[str] = context.list_expectation_suite_names()
except Exception as e:
toolkit.send_usage_message(
data_context=context, event=usage_event_end, success=False
)
raise e
if not suite_names:
toolkit.exit_with_failure_message_and_stats(
data_context=context,
usage_event=usage_event_end,
suppress_usage_message=False,
message="<red>No expectation suites found in the project.</red>",
)
if suite not in suite_names:
toolkit.exit_with_failure_message_and_stats(
data_context=context,
usage_event=usage_event_end,
suppress_usage_message=False,
message=f"<red>No expectation suite named {suite} found.</red>",
)
if not (
ctx.obj.assume_yes
or toolkit.confirm_proceed_or_exit(
exit_on_no=False, data_context=context, usage_stats_event=usage_event_end
)
):
cli_message(string=f"Suite `{suite}` was not deleted.")
sys.exit(0)
context.delete_expectation_suite(suite)
cli_message(string=f"Deleted the expectation suite named: {suite}")
toolkit.send_usage_message(
data_context=context, event=usage_event_end, success=True
)
def datasource_list(ctx):
"""List known datasources."""
display_not_implemented_message_and_exit()
context = ctx.obj.data_context
datasources = context.list_datasources()
datasource_count = len(datasources)
if datasource_count == 0:
list_intro_string = "No Datasources found"
else:
list_intro_string = _build_datasource_intro_string(datasource_count)
cli_message(list_intro_string)
for datasource in datasources:
cli_message("")
cli_message_dict(datasource)
toolkit.send_usage_message(
data_context=context, event="cli.datasource.list", success=True
)
def store_list(ctx):
"""List active Stores."""
context = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
try:
stores = context.list_active_stores()
cli_message(f"{len(stores)} active Stores found:")
for store in stores:
cli_message("")
cli_message_dict(store)
toolkit.send_usage_message(data_context=context,
event=usage_event_end,
success=True)
except Exception as e:
toolkit.exit_with_failure_message_and_stats(
context=context,
usage_event=usage_event_end,
message=f"<red>{e}</red>",
)
return
def store_list(ctx):
"""List known Stores."""
display_not_implemented_message_and_exit()
context = ctx.obj.data_context
try:
stores = context.list_stores()
if len(stores) == 0:
cli_message("No Stores found")
toolkit.send_usage_message(data_context=context,
event="cli.store.list",
success=True)
return
elif len(stores) == 1:
list_intro_string = "1 Store found:"
else:
list_intro_string = "{} Stores found:".format(len(stores))
cli_message(list_intro_string)
for store in stores:
cli_message("")
cli_message_dict(store)
toolkit.send_usage_message(data_context=context,
event="cli.store.list",
success=True)
except Exception as e:
toolkit.send_usage_message(data_context=context,
event="cli.store.list",
success=False)
raise e
def suite_list(ctx):
"""List existing Expectation Suites."""
context: DataContext = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
try:
suite_names: List[str] = context.list_expectation_suite_names()
except Exception as e:
toolkit.send_usage_message(data_context=context,
event=usage_event_end,
success=False)
raise e
suite_names_styled: List[str] = [
f" - <cyan>{suite_name}</cyan>" for suite_name in suite_names
]
if len(suite_names_styled) == 0:
cli_message(string="No Expectation Suites found")
toolkit.send_usage_message(data_context=context,
event=usage_event_end,
success=True)
return
list_intro_string: str
if len(suite_names_styled) == 1:
list_intro_string = "1 Expectation Suite found:"
else:
list_intro_string = f"{len(suite_names_styled)} Expectation Suites found:"
cli_message_list(string_list=suite_names_styled,
list_intro_string=list_intro_string)
toolkit.send_usage_message(data_context=context,
event=usage_event_end,
success=True)
def suite_list(ctx):
"""Lists available Expectation Suites."""
display_not_implemented_message_and_exit()
directory = toolkit.parse_cli_config_file_location(
config_file_location=ctx.obj.config_file_location).get("directory")
context = toolkit.load_data_context_with_error_handling(directory)
try:
suite_names = [
" - <cyan>{}</cyan>".format(suite_name)
for suite_name in context.list_expectation_suite_names()
]
if len(suite_names) == 0:
cli_message("No Expectation Suites found")
toolkit.send_usage_message(data_context=context,
event="cli.suite.list",
success=True)
return
elif len(suite_names) == 1:
list_intro_string = "1 Expectation Suite found:"
else:
list_intro_string = "{} Expectation Suites found:".format(
len(suite_names))
cli_message_list(suite_names, list_intro_string)
toolkit.send_usage_message(data_context=context,
event="cli.suite.list",
success=True)
except Exception as e:
toolkit.send_usage_message(data_context=context,
event="cli.suite.list",
success=False)
raise e
