def docs_list(ctx):
"""List known Data Docs sites."""
context = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
docs_sites_url_dicts = context.get_docs_sites_urls()
try:
if len(docs_sites_url_dicts) == 0:
cli_message("No Data Docs sites found")
else:
docs_sites_strings = [
" - <cyan>{}</cyan>: {}".format(
docs_site_dict["site_name"],
docs_site_dict.get("site_url") or
f"site configured but does not exist. Run the following command to build site: great_expectations "
f'docs build --site-name {docs_site_dict["site_name"]}',
) for docs_site_dict in docs_sites_url_dicts
]
list_intro_string = _build_intro_string(docs_sites_strings)
cli_message_list(docs_sites_strings, list_intro_string)
toolkit.send_usage_message(data_context=context,
event=usage_event_end,
success=True)
except Exception as e:
toolkit.exit_with_failure_message_and_stats(
context=context,
usage_event=usage_event_end,
message=f"<red>{e}</red>",
)
return
def clean_data_docs(ctx, site_name=None, all_sites=False):
"""Delete data docs"""
context = ctx.obj.data_context
if (site_name is None and all_sites is False) or (site_name and all_sites):
toolkit.exit_with_failure_message_and_stats(
context,
usage_event="cli.docs.clean",
message=
"<red>Please specify either --all to remove all sites or a specific site using --site_name</red>",
)
try:
# if site_name is None, context.clean_data_docs(site_name=site_name)
# will clean all sites.
context.clean_data_docs(site_name=site_name)
toolkit.send_usage_message(data_context=context,
event="cli.docs.clean",
success=True)
cli_message("<green>{}</green>".format("Cleaned data docs"))
except DataContextError as de:
toolkit.exit_with_failure_message_and_stats(
context,
usage_event="cli.docs.clean",
message=f"<red>{de}</red>",
)
def checkpoint_script(checkpoint, directory):
"""
Create a python script to run a checkpoint. (Experimental)
Checkpoints can be run directly without this script using the
`great_expectations checkpoint run` command.
This script is provided for those who wish to run checkpoints via python.
"""
context = toolkit.load_data_context_with_error_handling(directory)
usage_event = "cli.checkpoint.script"
# Attempt to load the checkpoint and deal with errors
_ = toolkit.load_checkpoint(context, checkpoint, usage_event)
script_name = f"run_{checkpoint}.py"
script_path = os.path.join(context.root_directory,
context.GE_UNCOMMITTED_DIR, script_name)
if os.path.isfile(script_path):
toolkit.exit_with_failure_message_and_stats(
context,
usage_event,
f"""<red>Warning! A script named {script_name} already exists and this command will not overwrite it.</red>
- Existing file path: {script_path}""",
)
_write_checkpoint_script_to_disk(context.root_directory, checkpoint,
script_path)
cli_message(
f"""<green>A python script was created that runs the checkpoint named: `{checkpoint}`</green>
- The script is located in `great_expectations/uncommitted/run_{checkpoint}.py`
- The script can be run with `python great_expectations/uncommitted/run_{checkpoint}.py`"""
)
send_usage_message(context, event=usage_event, success=True)
def checkpoint_run(checkpoint, directory):
"""Run a checkpoint. (Experimental)"""
context = toolkit.load_data_context_with_error_handling(
directory=directory, from_cli_upgrade_command=False)
usage_event = "cli.checkpoint.run"
checkpoint: Checkpoint = toolkit.load_checkpoint(
context,
checkpoint,
usage_event,
)
try:
results = checkpoint.run()
except Exception as e:
toolkit.exit_with_failure_message_and_stats(context, usage_event,
f"<red>{e}</red>")
if not results["success"]:
cli_message("Validation failed!")
send_usage_message(context, event=usage_event, success=True)
print_validation_operator_results_details(results)
sys.exit(1)
cli_message("Validation succeeded!")
send_usage_message(context, event=usage_event, success=True)
print_validation_operator_results_details(results)
sys.exit(0)
def checkpoint_run(ctx, checkpoint):
"""Run a Checkpoint."""
usage_event: str = "cli.checkpoint.run"
context: DataContext = ctx.obj.data_context
try:
result: CheckpointResult = toolkit.run_checkpoint(
context=context,
checkpoint_name=checkpoint,
usage_event=usage_event,
)
except Exception as e:
toolkit.exit_with_failure_message_and_stats(
context=context,
usage_event=usage_event,
message=f"<red>{e}</red>",
)
return
if not result["success"]:
cli_message(string="Validation failed!")
toolkit.send_usage_message(context, event=usage_event, success=True)
print_validation_operator_results_details(result=result)
sys.exit(1)
cli_message("Validation succeeded!")
toolkit.send_usage_message(context, event=usage_event, success=True)
print_validation_operator_results_details(result=result)
sys.exit(0)
def suite_delete(ctx, suite):
"""
Delete an expectation suite from the expectation store.
"""
display_not_implemented_message_and_exit()
usage_event = "cli.suite.delete"
directory = toolkit.parse_cli_config_file_location(
config_file_location=ctx.obj.config_file_location).get("directory")
context = toolkit.load_data_context_with_error_handling(directory)
suite_names = context.list_expectation_suite_names()
if not suite_names:
toolkit.exit_with_failure_message_and_stats(
context,
usage_event,
"</red>No expectation suites found in the project.</red>",
)
if suite not in suite_names:
toolkit.exit_with_failure_message_and_stats(
context, usage_event, f"No expectation suite named {suite} found.")
context.delete_expectation_suite(suite)
cli_message(f"Deleted the expectation suite named: {suite}")
toolkit.send_usage_message(data_context=context,
event=usage_event,
success=True)
def checkpoint_delete(ctx: click.Context, checkpoint: str) -> None:
"""Delete a Checkpoint."""
context: DataContext = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
try:
toolkit.delete_checkpoint(
context=context,
checkpoint_name=checkpoint,
usage_event=usage_event_end,
assume_yes=ctx.obj.assume_yes,
)
send_usage_message(
data_context=context,
event=usage_event_end,
success=True,
)
except Exception as e:
toolkit.exit_with_failure_message_and_stats(
data_context=context,
usage_event=usage_event_end,
message=f"<red>{e}</red>",
)
return
cli_message(f'Checkpoint "{checkpoint}" deleted.')
sys.exit(0)
def checkpoint_delete(ctx, checkpoint):
"""Delete a Checkpoint."""
usage_event: str = "cli.checkpoint.delete"
context: DataContext = ctx.obj.data_context
try:
toolkit.delete_checkpoint(
context=context,
checkpoint_name=checkpoint,
usage_event=usage_event,
assume_yes=ctx.obj.assume_yes,
)
toolkit.send_usage_message(context,
event="cli.checkpoint.delete",
success=True)
except Exception as e:
toolkit.exit_with_failure_message_and_stats(
context=context,
usage_event=usage_event,
message=f"<red>{e}</red>",
)
return
cli_message(f'Checkpoint "{checkpoint}" deleted.')
sys.exit(0)
def docs_clean(ctx, site_name=None, all_sites=False):
"""
Remove all files from a Data Docs site.
This is a useful first step if you wish to completely re-build a site from scratch.
"""
context = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
if (site_name is None and all_sites is False) or (site_name and all_sites):
toolkit.exit_with_failure_message_and_stats(
data_context=context,
usage_event=usage_event_end,
message=
"<red>Please specify either --all to clean all sites or a specific site using --site-name</red>",
)
try:
# if site_name is None, context.clean_data_docs(site_name=site_name)
# will clean all sites.
context.clean_data_docs(site_name=site_name)
toolkit.send_usage_message(data_context=context,
event=usage_event_end,
success=True)
cli_message("<green>{}</green>".format("Cleaned data docs"))
except DataContextError as de:
toolkit.exit_with_failure_message_and_stats(
data_context=context,
usage_event=usage_event_end,
message=f"<red>{de}</red>",
)
def docs_build(ctx, site_name=None, no_view=False):
"""Build Data Docs for a project."""
context: DataContext = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
if site_name is not None and site_name not in context.get_site_names():
toolkit.exit_with_failure_message_and_stats(
data_context=context,
usage_event=usage_event_end,
message=
f"<red>The specified site name `{site_name}` does not exist in this project.</red>",
)
if site_name is None:
sites_to_build = context.get_site_names()
else:
sites_to_build = [site_name]
build_docs(
context,
usage_stats_event=usage_event_end,
site_names=sites_to_build,
view=not no_view,
assume_yes=ctx.obj.assume_yes,
)
toolkit.send_usage_message(data_context=context,
event=usage_event_end,
success=True)
def datasource_list(ctx: click.Context) -> None:
"""List known Datasources."""
context = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
try:
datasources = context.list_datasources()
cli_message(_build_datasource_intro_string(datasources))
for datasource in datasources:
cli_message("")
cli_message_dict({
"name": datasource["name"],
"class_name": datasource["class_name"],
})
send_usage_message(
data_context=context,
event=usage_event_end,
success=True,
)
except Exception as e:
toolkit.exit_with_failure_message_and_stats(
data_context=context,
usage_event=usage_event_end,
message=f"<red>{e}</red>",
)
return
def _verify_checkpoint_does_not_exist(context: DataContext, checkpoint: str,
usage_event: str) -> None:
if checkpoint in context.list_checkpoints():
toolkit.exit_with_failure_message_and_stats(
context,
usage_event,
f"A checkpoint named `{checkpoint}` already exists. Please choose a new name.",
)
def _validate_at_least_one_suite_is_listed(context: DataContext, batch: dict,
checkpoint_file: str) -> None:
batch_kwargs = batch["batch_kwargs"]
suites = batch["expectation_suite_names"]
if not suites:
toolkit.exit_with_failure_message_and_stats(
context,
"cli.checkpoint.run",
f"""<red>A batch has no suites associated with it. At least one suite is required.
- Batch: {batch_kwargs}
- Please add at least one suite to your checkpoint file: {checkpoint_file}</red>""",
)
def _verify_checkpoint_does_not_exist(context: DataContext, checkpoint: str,
usage_event: str) -> None:
try:
if checkpoint in context.list_checkpoints():
toolkit.exit_with_failure_message_and_stats(
context,
usage_event,
f"A checkpoint named `{checkpoint}` already exists. Please choose a new name.",
)
except InvalidTopLevelConfigKeyError as e:
toolkit.exit_with_failure_message_and_stats(context, usage_event,
f"<red>{e}</red>")
def checkpoint_run(checkpoint, directory):
"""Run a checkpoint. (Experimental)"""
context = toolkit.load_data_context_with_error_handling(directory)
usage_event = "cli.checkpoint.run"
checkpoint_config = toolkit.load_checkpoint(context, checkpoint,
usage_event)
checkpoint_file = f"great_expectations/checkpoints/{checkpoint}.yml"
# TODO loading batches will move into DataContext eventually
batches_to_validate = []
for batch in checkpoint_config["batches"]:
_validate_at_least_one_suite_is_listed(context, batch, checkpoint_file)
batch_kwargs = batch["batch_kwargs"]
for suite_name in batch["expectation_suite_names"]:
suite = toolkit.load_expectation_suite(context, suite_name,
usage_event)
try:
batch = toolkit.load_batch(context, suite, batch_kwargs)
except (FileNotFoundError, SQLAlchemyError, OSError,
DataContextError) as e:
toolkit.exit_with_failure_message_and_stats(
context,
usage_event,
f"""<red>There was a problem loading a batch:
- Batch: {batch_kwargs}
- {e}
- Please verify these batch kwargs in the checkpoint file: `{checkpoint_file}`</red>""",
)
batches_to_validate.append(batch)
try:
results = context.run_validation_operator(
checkpoint_config["validation_operator_name"],
assets_to_validate=batches_to_validate,
# TODO prepare for new RunID - checkpoint name and timestamp
# run_id=RunID(checkpoint)
)
except DataContextError as e:
toolkit.exit_with_failure_message_and_stats(context, usage_event,
f"<red>{e}</red>")
if not results["success"]:
cli_message("Validation failed!")
send_usage_message(context, event=usage_event, success=True)
print_validation_operator_results_details(results)
sys.exit(1)
cli_message("Validation succeeded!")
send_usage_message(context, event=usage_event, success=True)
print_validation_operator_results_details(results)
sys.exit(0)
def suite_delete(ctx: click.Context, suite: str) -> None:
"""
Delete an Expectation Suite from the Expectation Store.
"""
context: DataContext = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
try:
suite_names: List[str] = context.list_expectation_suite_names()
except Exception as e:
send_usage_message(
data_context=context,
event=usage_event_end,
success=False,
)
raise e
if not suite_names:
toolkit.exit_with_failure_message_and_stats(
data_context=context,
usage_event=usage_event_end,
suppress_usage_message=False,
message="<red>No expectation suites found in the project.</red>",
)
if suite not in suite_names:
toolkit.exit_with_failure_message_and_stats(
data_context=context,
usage_event=usage_event_end,
suppress_usage_message=False,
message=f"<red>No expectation suite named {suite} found.</red>",
)
if not (
ctx.obj.assume_yes
or toolkit.confirm_proceed_or_exit(
exit_on_no=False, data_context=context, usage_stats_event=usage_event_end
)
):
cli_message(string=f"Suite `{suite}` was not deleted.")
sys.exit(0)
context.delete_expectation_suite(suite)
cli_message(string=f"Deleted the expectation suite named: {suite}")
send_usage_message(
data_context=context,
event=usage_event_end,
success=True,
)
def checkpoint_script(ctx: click.Context, checkpoint: str) -> None:
"""
Create a python script to run a Checkpoint.
Checkpoints can be run directly without this script using the
`great_expectations Checkpoint run` command.
This script is provided for those who wish to run Checkpoints via python.
"""
context: DataContext = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
toolkit.validate_checkpoint(context=context,
checkpoint_name=checkpoint,
usage_event=usage_event_end)
script_name: str = f"run_{checkpoint}.py"
script_path: str = os.path.join(context.root_directory,
context.GE_UNCOMMITTED_DIR, script_name)
if os.path.isfile(script_path):
toolkit.exit_with_failure_message_and_stats(
data_context=context,
usage_event=usage_event_end,
message=
f"""<red>Warning! A script named {script_name} already exists and this command will not overwrite it.</red>
- Existing file path: {script_path}""",
)
_write_checkpoint_script_to_disk(
context_directory=context.root_directory,
checkpoint_name=checkpoint,
script_path=script_path,
)
cli_message(
f"""<green>A python script was created that runs the Checkpoint named: `{checkpoint}`</green>
- The script is located in `great_expectations/uncommitted/run_{checkpoint}.py`
- The script can be run with `python great_expectations/uncommitted/run_{checkpoint}.py`"""
)
send_usage_message(
data_context=context,
event=usage_event_end,
success=True,
)
def suite_delete(suite, directory):
"""
Delete an expectation suite from the expectation store.
"""
usage_event = "cli.suite.delete"
context = toolkit.load_data_context_with_error_handling(directory)
suite_names = context.list_expectation_suite_names()
if not suite_names:
toolkit.exit_with_failure_message_and_stats(
context, usage_event,
"</red>No expectation suites found in the project.</red>")
if suite not in suite_names:
toolkit.exit_with_failure_message_and_stats(
context, usage_event, f"No expectation suite named {suite} found.")
context.delete_expectation_suite(suite)
cli_message(f"Deleted the expectation suite named: {suite}")
send_usage_message(data_context=context, event=usage_event, success=True)
def _checkpoint_new(ctx, checkpoint_name, jupyter):
context: DataContext = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
try:
_verify_checkpoint_does_not_exist(context, checkpoint_name, usage_event_end)
# Create notebook on disk
notebook_name = f"edit_checkpoint_{checkpoint_name}.ipynb"
notebook_file_path = _get_notebook_path(context, notebook_name)
checkpoint_new_notebook_renderer = CheckpointNewNotebookRenderer(
context=context, checkpoint_name=checkpoint_name
)
checkpoint_new_notebook_renderer.render_to_disk(
notebook_file_path=notebook_file_path
)
if not jupyter:
cli_message(
f"To continue editing this Checkpoint, run <green>jupyter notebook {notebook_file_path}</green>"
)
send_usage_message(
data_context=context,
event=usage_event_end,
success=True,
)
if jupyter:
cli_message(
"""<green>Because you requested to create a new Checkpoint, we'll open a notebook for you now to edit it!
If you wish to avoid this you can add the `--no-jupyter` flag.</green>\n\n"""
)
toolkit.launch_jupyter_notebook(notebook_file_path)
except Exception as e:
toolkit.exit_with_failure_message_and_stats(
data_context=context,
usage_event=usage_event_end,
message=f"<red>{e}</red>",
)
return
def datasource_new(ctx: click.Context, name: str, jupyter: bool) -> None:
"""Add a new Datasource to the data context."""
context: DataContext = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
try:
_datasource_new_flow(
context,
usage_event_end=usage_event_end,
datasource_name=name,
jupyter=jupyter,
)
except Exception as e:
toolkit.exit_with_failure_message_and_stats(
data_context=context,
usage_event=usage_event_end,
message=f"<red>{e}</red>",
)
return
def store_list(ctx):
"""List active Stores."""
context = ctx.obj.data_context
usage_event_end: str = ctx.obj.usage_event_end
try:
stores = context.list_active_stores()
cli_message(f"{len(stores)} active Stores found:")
for store in stores:
cli_message("")
cli_message_dict(store)
toolkit.send_usage_message(data_context=context,
event=usage_event_end,
success=True)
except Exception as e:
toolkit.exit_with_failure_message_and_stats(
context=context,
usage_event=usage_event_end,
message=f"<red>{e}</red>",
)
return
def checkpoint_run(checkpoint, directory):
"""Run a checkpoint. (Experimental)"""
usage_event = "cli.checkpoint.run"
context = toolkit.load_data_context_with_error_handling(
directory=directory, from_cli_upgrade_command=False)
ge_config_version = context.get_config().config_version
if ge_config_version >= 3:
cli_message(
f"""<red>The `checkpoint run` CLI command is not yet implemented for GE config versions >= 3.</red>"""
)
send_usage_message(context, usage_event, success=False)
sys.exit(1)
checkpoint: Checkpoint = toolkit.load_checkpoint(
context,
checkpoint,
usage_event,
)
try:
results = checkpoint.run()
except Exception as e:
toolkit.exit_with_failure_message_and_stats(context, usage_event,
f"<red>{e}</red>")
if not results["success"]:
cli_message("Validation failed!")
send_usage_message(context, event=usage_event, success=True)
print_validation_operator_results_details(results)
sys.exit(1)
cli_message("Validation succeeded!")
send_usage_message(context, event=usage_event, success=True)
print_validation_operator_results_details(results)
sys.exit(0)
