def get_runtime_kwargs(
self,
configuration: Optional[ExpectationConfiguration] = None,
runtime_configuration: dict = None,
):
if not configuration:
configuration = self.configuration
configuration = deepcopy(configuration)
if runtime_configuration:
configuration.kwargs.update(runtime_configuration)
success_kwargs = self.get_success_kwargs(configuration)
runtime_kwargs = {
key: configuration.kwargs.get(key, self.default_kwarg_values.get(key))
for key in self.runtime_keys
}
runtime_kwargs.update(success_kwargs)
runtime_kwargs["result_format"] = parse_result_format(
runtime_kwargs["result_format"]
)
return runtime_kwargs
def _validate(
self,
configuration: ExpectationConfiguration,
metrics: Dict,
runtime_configuration: dict = None,
execution_engine: ExecutionEngine = None,
):
if runtime_configuration:
result_format = runtime_configuration.get(
"result_format",
configuration.kwargs.get(
"result_format", self.default_kwarg_values.get("result_format")
),
)
else:
result_format = configuration.kwargs.get(
"result_format", self.default_kwarg_values.get("result_format")
)
mostly = self.get_success_kwargs().get(
"mostly", self.default_kwarg_values.get("mostly")
)
total_count = metrics.get("table.row_count")
null_count = metrics.get("column_values.nonnull.unexpected_count")
unexpected_count = metrics.get(self.map_metric + ".unexpected_count")
success = None
if total_count is None or null_count is None:
# Vacuously true
success = True
elif (total_count - null_count) != 0:
success_ratio = (total_count - unexpected_count - null_count) / (
total_count - null_count
)
success = success_ratio >= mostly
elif total_count == 0 or (total_count - null_count) == 0:
success = True
try:
nonnull_count = metrics.get("table.row_count") - metrics.get(
"column_values.nonnull.unexpected_count"
)
except TypeError:
nonnull_count = None
return _format_map_output(
result_format=parse_result_format(result_format),
success=success,
element_count=metrics.get("table.row_count"),
nonnull_count=nonnull_count,
unexpected_count=metrics.get(self.map_metric + ".unexpected_count"),
unexpected_list=metrics.get(self.map_metric + ".unexpected_values"),
unexpected_index_list=metrics.get(
self.map_metric + ".unexpected_index_list"
),
)
def get_validation_dependencies(
self,
configuration: Optional[ExpectationConfiguration] = None,
execution_engine: Optional[ExecutionEngine] = None,
runtime_configuration: Optional[dict] = None,
):
"""Returns the result format and metrics required to validate this Expectation using the provided result format."""
return {
"result_format": parse_result_format(
self.get_runtime_kwargs(
configuration=configuration,
runtime_configuration=runtime_configuration,
).get("result_format")
),
"metrics": dict(),
}
def _validate(
self,
configuration: ExpectationConfiguration,
metrics: Dict,
runtime_configuration: dict = None,
execution_engine: ExecutionEngine = None,
):
result_format = self.get_result_format(
configuration=configuration, runtime_configuration=runtime_configuration
)
mostly = self.get_success_kwargs().get(
"mostly", self.default_kwarg_values.get("mostly")
)
total_count = metrics.get("table.row_count")
unexpected_count = metrics.get(f"{self.map_metric}.unexpected_count")
if total_count is None or total_count == 0:
# Vacuously true
success = True
else:
success_ratio = (total_count - unexpected_count) / total_count
success = success_ratio >= mostly
nonnull_count = None
return _format_map_output(
result_format=parse_result_format(result_format),
success=success,
element_count=metrics.get("table.row_count"),
nonnull_count=nonnull_count,
unexpected_count=metrics.get(f"{self.map_metric}.unexpected_count"),
unexpected_list=metrics.get(f"{self.map_metric}.unexpected_values"),
unexpected_index_list=metrics.get(
f"{self.map_metric}.unexpected_index_list"
),
)
