def validation_operator_list(directory):
"""Lists the names of the validation operators configured in the project."""
try:
context = DataContext(directory)
except ge_exceptions.ConfigNotFoundError as err:
cli_message("<red>{}</red>".format(err.message))
return
suite_names = context.list_validation_operator_names()
if len(suite_names) == 0:
cli_message("No validation operators are configured in the project")
return
if len(suite_names) == 1:
cli_message("1 validation operator found:")
if len(suite_names) > 1:
cli_message("{} validation operators found:".format(len(suite_names)))
for name in suite_names:
cli_message("\t{}".format(name))
def validation_operator_run(name, run_name, validation_config_file, suite,
directory):
# Note though the long lines here aren't pythonic, they look best if Click does the line wraps.
"""
Run a validation operator against some data.
There are two modes to run this command:
1. Interactive (good for development):
Specify the name of the validation operator using the --name argument
and the name of the expectation suite using the --suite argument.
The cli will help you specify the batch of data that you want to
validate interactively.
2. Non-interactive (good for production):
Use the `--validation_config_file` argument to specify the path of the validation configuration JSON file. This file can be used to instruct a validation operator to validate multiple batches of data and use multiple expectation suites to validate each batch.
Learn how to create a validation config file here:
https://great-expectations.readthedocs.io/en/latest/command_line.html#great-expectations-validation-operator-run-validation-config-file-validation-config-file-path
This command exits with 0 if the validation operator ran and the "success" attribute in its return object is True. Otherwise, the command exits with 1.
To learn more about validation operators, go here:
https://great-expectations.readthedocs.io/en/latest/features/validation.html#validation-operators
"""
try:
context = DataContext(directory)
except ge_exceptions.ConfigNotFoundError as err:
cli_message("Failed to process <red>{}</red>".format(err.message))
sys.exit(1)
try:
if validation_config_file is not None:
try:
with open(validation_config_file) as f:
validation_config = json.load(f)
except (OSError, json_parse_exception) as e:
cli_message(
f"Failed to process the --validation_config_file argument: <red>{e}</red>"
)
send_usage_message(
data_context=context,
event="cli.validation_operator.run",
success=False,
)
sys.exit(1)
validation_config_error_message = _validate_valdiation_config(
validation_config)
if validation_config_error_message is not None:
cli_message(
"<red>The validation config in {:s} is misconfigured: {:s}</red>"
.format(validation_config_file,
validation_config_error_message))
send_usage_message(
data_context=context,
event="cli.validation_operator.run",
success=False,
)
sys.exit(1)
else:
if suite is None:
cli_message("""
Please use --suite argument to specify the name of the expectation suite.
Call `great_expectation suite list` command to list the expectation suites in your project.
""")
send_usage_message(
data_context=context,
event="cli.validation_operator.run",
success=False,
)
sys.exit(0)
suite = toolkit.load_expectation_suite(
context, suite, "cli.validation_operator.run")
if name is None:
cli_message("""
Please use --name argument to specify the name of the validation operator.
Call `great_expectation validation-operator list` command to list the operators in your project.
""")
send_usage_message(
data_context=context,
event="cli.validation_operator.run",
success=False,
)
sys.exit(1)
else:
if name not in context.list_validation_operator_names():
cli_message(f"""
Could not find a validation operator {name}.
Call `great_expectation validation-operator list` command to list the operators in your project.
""")
send_usage_message(
data_context=context,
event="cli.validation_operator.run",
success=False,
)
sys.exit(1)
batch_kwargs = None
cli_message("""
Let us help you specify the batch of data your want the validation operator to validate."""
)
try:
data_source = toolkit.select_datasource(context)
except ValueError as ve:
cli_message("<red>{}</red>".format(ve))
send_usage_message(
data_context=context,
event="cli.validation_operator.run",
success=False,
)
sys.exit(1)
if not data_source:
cli_message("<red>No datasources found in the context.</red>")
send_usage_message(
data_context=context,
event="cli.validation_operator.run",
success=False,
)
sys.exit(1)
if batch_kwargs is None:
(
datasource_name,
batch_kwargs_generator,
data_asset,
batch_kwargs,
) = get_batch_kwargs(
context,
datasource_name=data_source.name,
batch_kwargs_generator_name=None,
data_asset_name=None,
additional_batch_kwargs=None,
)
validation_config = {
"validation_operator_name":
name,
"batches": [{
"batch_kwargs":
batch_kwargs,
"expectation_suite_names": [suite.expectation_suite_name],
}],
}
try:
validation_operator_name = validation_config[
"validation_operator_name"]
batches_to_validate = []
for entry in validation_config["batches"]:
for expectation_suite_name in entry["expectation_suite_names"]:
batch = context.get_batch(entry["batch_kwargs"],
expectation_suite_name)
batches_to_validate.append(batch)
if run_name is None:
run_name = datetime.datetime.now(
datetime.timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ")
run_id = RunIdentifier(run_name=run_name)
if suite is None:
results = context.run_validation_operator(
validation_operator_name,
assets_to_validate=batches_to_validate,
run_id=run_id,
)
else:
if suite.evaluation_parameters is None:
results = context.run_validation_operator(
validation_operator_name,
assets_to_validate=batches_to_validate,
run_id=run_id,
)
else:
results = context.run_validation_operator(
validation_operator_name,
assets_to_validate=batches_to_validate,
run_id=run_id,
evaluation_parameters=suite.evaluation_parameters,
)
except (ge_exceptions.DataContextError, OSError, SQLAlchemyError) as e:
cli_message("<red>{}</red>".format(e))
send_usage_message(data_context=context,
event="cli.validation_operator.run",
success=False)
sys.exit(1)
if not results["success"]:
cli_message("Validation failed!")
send_usage_message(data_context=context,
event="cli.validation_operator.run",
success=True)
sys.exit(1)
else:
cli_message("Validation succeeded!")
send_usage_message(data_context=context,
event="cli.validation_operator.run",
success=True)
sys.exit(0)
except Exception as e:
send_usage_message(data_context=context,
event="cli.validation_operator.run",
success=False)
raise e
