def load_checkpoint(
context: DataContext,
checkpoint_name: str,
usage_event: str,
) -> Union[Checkpoint, LegacyCheckpoint]:
"""Load a checkpoint or raise helpful errors."""
try:
checkpoint: Union[Checkpoint, LegacyCheckpoint] = context.get_checkpoint(
name=checkpoint_name
)
return checkpoint
except (
ge_exceptions.CheckpointNotFoundError,
ge_exceptions.InvalidCheckpointConfigError,
):
exit_with_failure_message_and_stats(
context,
usage_event,
f"""\
<red>Could not find checkpoint `{checkpoint_name}`.</red> Try running:
- `<green>great_expectations checkpoint list</green>` to verify your checkpoint exists
- `<green>great_expectations checkpoint new</green>` to configure a new checkpoint""",
)
except ge_exceptions.CheckpointError as e:
exit_with_failure_message_and_stats(context, usage_event, f"<red>{e}</red>")
def run(
self,
checkpoint_name: str = None,
ge_checkpoint: Checkpoint = None,
checkpoint_kwargs: dict = None,
context: ge.DataContext = None,
assets_to_validate: list = None,
batch_kwargs: dict = None,
expectation_suite_name: str = None,
context_root_dir: str = None,
runtime_environment: Optional[dict] = None,
run_name: str = None,
run_info_at_end: bool = True,
disable_markdown_artifact: bool = False,
validation_operator: str = "action_list_operator",
evaluation_parameters: Optional[dict] = None,
):
"""
Task run method.
Args:
- checkpoint_name (str, optional): the name of a pre-configured checkpoint; should match the
filename of the checkpoint without the extension. Either checkpoint_name or
checkpoint_config is required when using the Great Expectations v3 API.
- ge_checkpoint (Checkpoint, optional): an in-memory GE `Checkpoint` object used to perform
validation. If not provided then `checkpoint_name` will be used to load the specified
checkpoint.
- checkpoint_kwargs (Dict, optional): A dictionary whose keys match the parameters of
`CheckpointConfig` which can be used to update and populate the task's Checkpoint at
runtime.
- context (DataContext, optional): an in-memory GE `DataContext` object. e.g.
`ge.data_context.DataContext()` If not provided then `context_root_dir` will be used to
look for one.
- assets_to_validate (list, optional): A list of assets to validate when running the
validation operator. Only used in the Great Expectations v2 API
- batch_kwargs (dict, optional): a dictionary of batch kwargs to be used when validating
assets. Only used in the Great Expectations v2 API
- expectation_suite_name (str, optional): the name of an expectation suite to be used when
validating assets. Only used in the Great Expectations v2 API
- context_root_dir (str, optional): the absolute or relative path to the directory holding
your `great_expectations.yml`
- runtime_environment (dict, optional): a dictionary of great expectation config key-value
pairs to overwrite your config in `great_expectations.yml`
- run_name (str, optional): the name of this Great Expectation validation run; defaults to
the task slug
- run_info_at_end (bool, optional): add run info to the end of the artifact generated by this
task. Defaults to `True`.
- disable_markdown_artifact (bool, optional): toggle the posting of a markdown artifact from
this tasks. Defaults to `False`.
- evaluation_parameters (Optional[dict], optional): the evaluation parameters to use when
running validation. For more information, see
[example](https://docs.prefect.io/api/latest/tasks/great_expectations.html#rungreatexpectationsvalidation)
and
[docs](https://docs.greatexpectations.io/en/latest/reference/core_concepts/evaluation_parameters.html).
- validation_operator (str, optional): configure the actions to be executed after running
validation. Defaults to `action_list_operator`.
Raises:
- 'signals.FAIL' if the validation was not a success
Returns:
- result
('great_expectations.validation_operators.types.validation_operator_result.ValidationOperatorResult'):
The Great Expectations metadata returned from the validation if the v2 (batch_kwargs) API
is used.
('great_expectations.checkpoint.checkpoint.CheckpointResult'):
The Great Expectations metadata returned from running the provided checkpoint if a
checkpoint name is provided.
"""
if version.parse(ge.__version__) < version.parse("0.13.8"):
self.logger.warning(
f"You are using great_expectations version {ge.__version__} which may cause"
"errors in this task. Please upgrade great_expections to 0.13.8 or later."
)
runtime_environment = runtime_environment or dict()
checkpoint_kwargs = checkpoint_kwargs or dict()
# Load context if not provided directly
if not context:
context = ge.DataContext(
context_root_dir=context_root_dir,
runtime_environment=runtime_environment,
)
# Check that the parameters are mutually exclusive
if (sum(
bool(x) for x in [
(expectation_suite_name and batch_kwargs),
assets_to_validate,
checkpoint_name,
ge_checkpoint,
]) != 1):
raise ValueError(
"Exactly one of expectation_suite_name + batch_kwargs, assets_to_validate, "
"checkpoint_name, or ge_checkpoint is required to run validation."
)
results = None
# If there is a checkpoint or checkpoint name provided, run the checkpoint.
# Checkpoints are the preferred deployment of validation configuration.
if ge_checkpoint or checkpoint_name:
ge_checkpoint = ge_checkpoint or context.get_checkpoint(
checkpoint_name)
results = ge_checkpoint.run(
evaluation_parameters=evaluation_parameters,
run_id={
"run_name": run_name or prefect.context.get("task_slug")
},
**checkpoint_kwargs,
)
else:
# If assets are not provided directly through `assets_to_validate` then they need be loaded
# get batch from `batch_kwargs` and `expectation_suite_name`
if not assets_to_validate:
assets_to_validate = [
context.get_batch(batch_kwargs, expectation_suite_name)
]
# Run validation operator
results = context.run_validation_operator(
validation_operator,
assets_to_validate=assets_to_validate,
run_id={
"run_name": run_name or prefect.context.get("task_slug")
},
evaluation_parameters=evaluation_parameters,
)
# Generate artifact markdown
if not disable_markdown_artifact:
validation_results_page_renderer = (
ge.render.renderer.ValidationResultsPageRenderer(
run_info_at_end=run_info_at_end))
rendered_content_list = validation_results_page_renderer.render_validation_operator_result(
# This also works with a CheckpointResult because of duck typing.
# The passed in object needs a list_validation_results method that
# returns a list of ExpectationSuiteValidationResult.
validation_operator_result=results)
markdown_artifact = " ".join(
ge.render.view.DefaultMarkdownPageView().render(
rendered_content_list))
create_markdown_artifact(markdown_artifact)
if results.success is False:
raise signals.FAIL(result=results)
return results
Data are validated by use of the `ActionListValidationOperator` which is
configured by default. The default configuration of this Validation Operator
saves validation results to your results store and then updates Data Docs.
This makes viewing validation results easy for you and your team.
Usage:
- Run this file: `python great_expectations/uncommitted/run_{0}.py`.
- This can be run manually or via a scheduler such as cron.
- If your pipeline runner supports python snippets you can paste this into your
pipeline.
"""
import sys
from great_expectations import DataContext
# checkpoint configuration
context = DataContext("{1}")
checkpoint = context.get_checkpoint("{0}")
# run the Checkpoint
results = checkpoint.run()
# take action based on results
if not results["success"]:
print("Validation failed!")
sys.exit(1)
print("Validation succeeded!")
sys.exit(0)
