def checkpoint_run(checkpoint, directory):
"""Run a checkpoint. (Experimental)"""
context = toolkit.load_data_context_with_error_handling(directory)
usage_event = "cli.checkpoint.run"
checkpoint_config = toolkit.load_checkpoint(context, checkpoint,
usage_event)
checkpoint_file = f"great_expectations/checkpoints/{checkpoint}.yml"
# TODO loading batches will move into DataContext eventually
batches_to_validate = []
for batch in checkpoint_config["batches"]:
_validate_at_least_one_suite_is_listed(context, batch, checkpoint_file)
batch_kwargs = batch["batch_kwargs"]
for suite_name in batch["expectation_suite_names"]:
suite = toolkit.load_expectation_suite(context, suite_name,
usage_event)
try:
batch = toolkit.load_batch(context, suite, batch_kwargs)
except (FileNotFoundError, SQLAlchemyError, OSError,
DataContextError) as e:
toolkit.exit_with_failure_message_and_stats(
context,
usage_event,
f"""<red>There was a problem loading a batch:
- Batch: {batch_kwargs}
- {e}
- Please verify these batch kwargs in the checkpoint file: `{checkpoint_file}`</red>""",
)
batches_to_validate.append(batch)
try:
results = context.run_validation_operator(
checkpoint_config["validation_operator_name"],
assets_to_validate=batches_to_validate,
# TODO prepare for new RunID - checkpoint name and timestamp
# run_id=RunID(checkpoint)
)
except DataContextError as e:
toolkit.exit_with_failure_message_and_stats(context, usage_event,
f"<red>{e}</red>")
if not results["success"]:
cli_message("Validation failed!")
send_usage_message(context, event=usage_event, success=True)
print_validation_operator_results_details(results)
sys.exit(1)
cli_message("Validation succeeded!")
send_usage_message(context, event=usage_event, success=True)
print_validation_operator_results_details(results)
sys.exit(0)
def _suite_edit(suite, datasource, directory, jupyter, batch_kwargs, usage_event):
batch_kwargs_json = batch_kwargs
batch_kwargs = None
context = load_data_context_with_error_handling(directory)
try:
suite = load_expectation_suite(context, suite)
citations = suite.get_citations(require_batch_kwargs=True)
if batch_kwargs_json:
try:
batch_kwargs = json.loads(batch_kwargs_json)
if datasource:
batch_kwargs["datasource"] = datasource
_batch = toolkit.load_batch(context, suite, batch_kwargs)
except json_parse_exception as je:
cli_message(
"<red>Please check that your batch_kwargs are valid JSON.\n{}</red>".format(
je
)
)
send_usage_message(
data_context=context, event=usage_event, success=False
)
sys.exit(1)
except ge_exceptions.DataContextError:
cli_message(
"<red>Please check that your batch_kwargs are able to load a batch.</red>"
)
send_usage_message(
data_context=context, event=usage_event, success=False
)
sys.exit(1)
except ValueError as ve:
cli_message(
"<red>Please check that your batch_kwargs are able to load a batch.\n{}</red>".format(
ve
)
)
send_usage_message(
data_context=context, event=usage_event, success=False
)
sys.exit(1)
elif citations:
citation = citations[-1]
batch_kwargs = citation.get("batch_kwargs")
if not batch_kwargs:
cli_message(
"""
A batch of data is required to edit the suite - let's help you to specify it."""
)
additional_batch_kwargs = None
try:
data_source = select_datasource(context, datasource_name=datasource)
except ValueError as ve:
cli_message("<red>{}</red>".format(ve))
send_usage_message(
data_context=context, event=usage_event, success=False
)
sys.exit(1)
if not data_source:
cli_message("<red>No datasources found in the context.</red>")
send_usage_message(
data_context=context, event=usage_event, success=False
)
sys.exit(1)
if batch_kwargs is None:
(
datasource_name,
batch_kwargs_generator,
data_asset,
batch_kwargs,
) = get_batch_kwargs(context, datasource_name=data_source.name,
additional_batch_kwargs=additional_batch_kwargs)
notebook_name = "edit_{}.ipynb".format(suite.expectation_suite_name)
notebook_path = _get_notebook_path(context, notebook_name)
SuiteEditNotebookRenderer().render_to_disk(suite, notebook_path, batch_kwargs)
if not jupyter:
cli_message(
f"To continue editing this suite, run <green>jupyter notebook {notebook_path}</green>"
)
payload = edit_expectation_suite_usage_statistics(
data_context=context, expectation_suite_name=suite.expectation_suite_name
)
send_usage_message(
data_context=context, event=usage_event, event_payload=payload, success=True
)
if jupyter:
toolkit.launch_jupyter_notebook(notebook_path)
except Exception as e:
send_usage_message(data_context=context, event=usage_event, success=False)
raise e