def critical_suite():
"""
This hand made fixture has a wide range of expectations, and has a mix of
metadata including an SampleExpectationsDatasetProfiler entry.
"""
schema = ExpectationSuiteSchema(strict=True)
critical_suite = {
"expectation_suite_name": "critical",
"meta": {"great_expectations.__version__": "0.7.10"},
"expectations": [
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {"column": "npi"},
"meta": {
"question": True,
"Notes": "There are empty strings that should probably be nulls",
'SampleExpectationsDatasetProfiler': {
'confidence': 'very low'
}
},
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {"column": "provider_type"},
},
],
"data_asset_type": "Dataset",
}
return schema.loads(json.dumps(critical_suite)).data
class ExpectationsStore(Store):
_key_class = ExpectationSuiteIdentifier
def __init__(self, store_backend=None, runtime_environment=None):
self._expectationSuiteSchema = ExpectationSuiteSchema()
if store_backend is not None:
store_backend_module_name = store_backend.get("module_name", "great_expectations.data_context.store")
store_backend_class_name = store_backend.get("class_name", "InMemoryStoreBackend")
store_backend_class = load_class(store_backend_class_name, store_backend_module_name)
if issubclass(store_backend_class, TupleStoreBackend):
# Provide defaults for this common case
store_backend["filepath_suffix"] = store_backend.get("filepath_suffix", ".json")
elif issubclass(store_backend_class, DatabaseStoreBackend):
# Provide defaults for this common case
store_backend["table_name"] = store_backend.get("table_name", "ge_expectations_store")
store_backend["key_columns"] = store_backend.get(
"key_columns", ["expectation_suite_name"]
)
super(ExpectationsStore, self).__init__(store_backend=store_backend, runtime_environment=runtime_environment)
def serialize(self, key, value):
return self._expectationSuiteSchema.dumps(value)
def deserialize(self, key, value):
return self._expectationSuiteSchema.loads(value)
class ExpectationsStore(Store):
_key_class = ExpectationSuiteIdentifier
def __init__(self, store_backend=None, runtime_environment=None):
self._expectationSuiteSchema = ExpectationSuiteSchema()
if store_backend is not None:
store_backend_module_name = store_backend.get("module_name", "great_expectations.data_context.store")
store_backend_class_name = store_backend.get("class_name", "InMemoryStoreBackend")
verify_dynamic_loading_support(module_name=store_backend_module_name)
store_backend_class = load_class(store_backend_class_name, store_backend_module_name)
# Store Backend Class was loaded successfully; verify that it is of a correct subclass.
if issubclass(store_backend_class, TupleStoreBackend):
# Provide defaults for this common case
store_backend["filepath_suffix"] = store_backend.get("filepath_suffix", ".json")
elif issubclass(store_backend_class, DatabaseStoreBackend):
# Provide defaults for this common case
store_backend["table_name"] = store_backend.get("table_name", "ge_expectations_store")
store_backend["key_columns"] = store_backend.get(
"key_columns", ["expectation_suite_name"]
)
super().__init__(store_backend=store_backend, runtime_environment=runtime_environment)
def serialize(self, key, value):
return self._expectationSuiteSchema.dumps(value, indent=2, sort_keys=True)
def deserialize(self, key, value):
return self._expectationSuiteSchema.loads(value)
def warning_suite():
"""
This hand made fixture has a wide range of expectations, and has a mix of
metadata including SampleExpectationsDatasetProfiler entries.
"""
schema = ExpectationSuiteSchema(strict=True)
warning_suite = {
"expectation_suite_name": "warning",
"meta": {"great_expectations.__version__": "0.8.4.post0"},
"expectations": [
{
"expectation_type": "expect_table_row_count_to_be_between",
"kwargs": {"min_value": 800000, "max_value": 1200000},
},
{
"expectation_type": "expect_table_column_count_to_equal",
"kwargs": {"value": 71},
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {"column": "npi"},
"meta": {
'SampleExpectationsDatasetProfiler': {
'confidence': 'very low'
}
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {"column": "provider_type"},
"meta": {
'SampleExpectationsDatasetProfiler': {
'confidence': 'very low'
}
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {"column": "nppes_provider_last_org_name"},
},
{
"expectation_type": "expect_column_values_to_be_in_set",
"kwargs": {
"column": "nppes_provider_gender",
"value_set": ["M", "F", ""],
},
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {"column": "nppes_entity_code"},
},
{
"expectation_type": "expect_column_values_to_be_in_set",
"kwargs": {"column": "nppes_entity_code", "value_set": ["I", "O"]},
},
{
"expectation_type": "expect_column_kl_divergence_to_be_less_than",
"kwargs": {
"column": "nppes_entity_code",
"partition_object": {
"values": ["I", "O"],
"weights": [0.9431769750233306, 0.056823024976669335],
},
"threshold": 0.1,
},
},
{
"expectation_type": "expect_column_values_to_be_in_set",
"kwargs": {
"column": "nppes_provider_state",
"value_set": [
"AL",
"AK",
"AZ",
"AR",
"CA",
"CO",
"CT",
"DE",
"FL",
"GA",
"HI",
"ID",
"IL",
"IN",
"IA",
"KS",
"KY",
"LA",
"ME",
"MD",
"MA",
"MI",
"MN",
"MS",
"MO",
"MT",
"NE",
"NV",
"NH",
"NJ",
"NM",
"NY",
"NC",
"ND",
"OH",
"OK",
"OR",
"PA",
"RI",
"SC",
"SD",
"TN",
"TX",
"UT",
"VT",
"VA",
"WA",
"WV",
"WI",
"WY",
"DC",
"PR",
"AE",
"VI",
],
"mostly": 0.999,
},
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {"column": "medicare_participation_indicator"},
},
{
"expectation_type": "expect_column_values_to_be_in_set",
"kwargs": {
"column": "medicare_participation_indicator",
"value_set": ["Y", "N"],
},
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {"column": "number_of_hcpcs"},
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "number_of_hcpcs",
"min_value": 0,
"max_value": 500,
"mostly": 0.999,
},
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {"column": "total_unique_benes"},
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "total_unique_benes",
"min_value": 0,
"max_value": 2000,
"mostly": 0.95,
},
},
{
"expectation_type": "expect_column_values_to_be_null",
"kwargs": {"column": "med_suppress_indicator", "mostly": 0.85},
},
{
"expectation_type": "expect_column_values_to_be_in_set",
"kwargs": {"column": "med_suppress_indicator", "value_set": ["#", "*"]},
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "beneficiary_average_age",
"min_value": 40,
"max_value": 90,
"mostly": 0.995,
},
},
{
"expectation_type": "expect_column_kl_divergence_to_be_less_than",
"kwargs": {
"column": "beneficiary_average_age",
"partition_object": {
"bins": [8, 16.5, 25, 33.5, 42, 50.5, 59, 67.5, 76, 84.5, 93],
"weights": [
0.00025259576594384474,
0.00013318685840675451,
0.0009653750909344757,
0.0012363414580378728,
0.01081660996274442,
0.030813927854975127,
0.13495227317818748,
0.6919590041664524,
0.1244213260634741,
0.004449359600843578,
],
},
"threshold": 0.9,
},
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "total_submitted_chrg_amt",
"min_value": 2000,
"max_value": 5000000,
"mostly": 0.98,
},
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {"column": "nppes_provider_first_name", "mostly": 0.9},
},
{
"expectation_type": "expect_column_values_to_match_regex",
"kwargs": {
"column": "nppes_provider_zip",
"regex": "^\\d*$",
"mostly": 0.999,
},
},
],
"data_asset_type": "Dataset",
}
return schema.loads(json.dumps(warning_suite)).data
class ExpectationsStore(Store):
"""
An Expectations Store provides a way to store Expectation Suites accessible to a Data Context.
--ge-feature-maturity-info--
id: expectations_store_git
title: Expectation Store - Git
icon:
short_description: Store Expectations in Git
description: Use a git repository to store expectation suites.
how_to_guide_url: https://docs.greatexpectations.io/en/latest/how_to_guides/configuring_metadata_stores/how_to_configure_an_expectation_store_on_a_filesystem.html#additional-notes
maturity: Production
maturity_details:
api_stability: Stable
implementation_completeness: Complete
unit_test_coverage: Complete
integration_infrastructure_test_coverage: N/A
documentation_completeness: Complete
bug_risk: Low
id: expectations_store_filesystem
title: Expectation Store - Filesystem
icon:
short_description: Filesystem-based Expectations Store
description: Filesystem-based Expectations Store
how_to_guide_url: https://docs.greatexpectations.io/en/latest/how_to_guides/configuring_metadata_stores/how_to_configure_an_expectation_store_on_a_filesystem.html
maturity: Production
maturity_details:
api_stability: Stable
implementation_completeness: Complete
unit_test_coverage: Complete
integration_infrastructure_test_coverage: N/A
documentation_completeness: Complete
bug_risk: Low
id: expectations_store_s3
title: Expectation Store - S3
icon:
short_description: S3
description: Use an Amazon Web Services S3 bucket to store expectations.
how_to_guide_url: https://docs.greatexpectations.io/en/latest/how_to_guides/configuring_metadata_stores/how_to_configure_an_expectation_store_in_amazon_s3.html
maturity: Beta
maturity_details:
api_stability: Stable
implementation_completeness: Complete
unit_test_coverage: Complete
integration_infrastructure_test_coverage: Minimal
documentation_completeness: Complete
bug_risk: Low
id: expectations_store_gcs
title: Expectation Store - GCS
icon:
short_description: Cloud Storage
description: Use a Google Cloud Platform Cloud Storage bucket to store expectations.
how_to_guide_url: https://docs.greatexpectations.io/en/latest/how_to_guides/configuring_metadata_stores/how_to_configure_an_expectation_store_in_gcs.html
maturity: Beta
maturity_details:
api_stability: Stable
implementation_completeness: Complete
unit_test_coverage: Complete
integration_infrastructure_test_coverage: Minimal
documentation_completeness: Partial
bug_risk: Low
id: expectations_store_azure_blob_storage
title: Expectation Store - Azure
icon:
short_description: Azure Blob Storage
description: Use Microsoft Azure Blob Storage to store expectations.
how_to_guide_url: https://docs.greatexpectations.io/en/latest/how_to_guides/configuring_metadata_stores/how_to_configure_an_expectation_store_in_azure_blob_storage.html
maturity: N/A
maturity_details:
api_stability: Stable
implementation_completeness: Minimal
unit_test_coverage: Minimal
integration_infrastructure_test_coverage: Minimal
documentation_completeness: Minimal
bug_risk: Moderate
--ge-feature-maturity-info--
"""
_key_class = ExpectationSuiteIdentifier
def __init__(self, store_backend=None, runtime_environment=None):
self._expectationSuiteSchema = ExpectationSuiteSchema()
if store_backend is not None:
store_backend_module_name = store_backend.get(
"module_name", "great_expectations.data_context.store"
)
store_backend_class_name = store_backend.get(
"class_name", "InMemoryStoreBackend"
)
verify_dynamic_loading_support(module_name=store_backend_module_name)
store_backend_class = load_class(
store_backend_class_name, store_backend_module_name
)
# Store Backend Class was loaded successfully; verify that it is of a correct subclass.
if issubclass(store_backend_class, TupleStoreBackend):
# Provide defaults for this common case
store_backend["filepath_suffix"] = store_backend.get(
"filepath_suffix", ".json"
)
elif issubclass(store_backend_class, DatabaseStoreBackend):
# Provide defaults for this common case
store_backend["table_name"] = store_backend.get(
"table_name", "ge_expectations_store"
)
store_backend["key_columns"] = store_backend.get(
"key_columns", ["expectation_suite_name"]
)
super().__init__(
store_backend=store_backend, runtime_environment=runtime_environment
)
def remove_key(self, key):
return self.store_backend.remove_key(key)
def serialize(self, key, value):
return self._expectationSuiteSchema.dumps(value, indent=2, sort_keys=True)
def deserialize(self, key, value):
return self._expectationSuiteSchema.loads(value)
class ExpectationsStore(Store):
"""
An Expectations Store provides a way to store Expectation Suites accessible to a Data Context.
.. admonition:: Feature Maturity
id: expectations_store_git
title: Expectation Store - Git
icon:
short_description:
description:
how_to_guide_url:
maturity: Production
maturity_details:
api_stability: Stable
implementation_completeness: Complete
unit_test_coverage: Complete
integration_infrastructure_test_coverage: N/A
documentation_completeness: Complete
bug_risk: Low
id: expectations_store_filesystem
title: Expectation Store - Filesystem
icon:
short_description:
description:
how_to_guide_url:
maturity: Production
maturity_details:
api_stability: Stable
implementation_completeness: Complete
unit_test_coverage: Complete
integration_infrastructure_test_coverage: N/A
documentation_completeness: Complete
bug_risk: Low
id: expectations_store_s3
title: Expectation Store - S3
icon:
short_description:
description:
how_to_guide_url:
maturity: Beta
maturity_details:
api_stability: Stable
implementation_completeness: Complete
unit_test_coverage: Complete
integration_infrastructure_test_coverage: Minimal
documentation_completeness: Complete
bug_risk: Low
id: expectations_store_gcs
title: Expectation Store - GCS
icon:
short_description:
description:
how_to_guide_url:
maturity: Beta
maturity_details:
api_stability: Stable
implementation_completeness: Complete
unit_test_coverage: Complete
integration_infrastructure_test_coverage: Minimal
documentation_completeness: Partial
bug_risk: Low
id: expectations_store_azure_blob_storage
title: Expectation Store - Azure
icon:
short_description:
description:
how_to_guide_url:
maturity: N/A
maturity_details:
api_stability: Stable
implementation_completeness: Minimal
unit_test_coverage: Minimal
integration_infrastructure_test_coverage: Minimal
documentation_completeness: Minimal
bug_risk: Moderate
"""
_key_class = ExpectationSuiteIdentifier
def __init__(self, store_backend=None, runtime_environment=None):
self._expectationSuiteSchema = ExpectationSuiteSchema()
if store_backend is not None:
store_backend_module_name = store_backend.get(
"module_name", "great_expectations.data_context.store")
store_backend_class_name = store_backend.get(
"class_name", "InMemoryStoreBackend")
verify_dynamic_loading_support(
module_name=store_backend_module_name)
store_backend_class = load_class(store_backend_class_name,
store_backend_module_name)
# Store Backend Class was loaded successfully; verify that it is of a correct subclass.
if issubclass(store_backend_class, TupleStoreBackend):
# Provide defaults for this common case
store_backend["filepath_suffix"] = store_backend.get(
"filepath_suffix", ".json")
elif issubclass(store_backend_class, DatabaseStoreBackend):
# Provide defaults for this common case
store_backend["table_name"] = store_backend.get(
"table_name", "ge_expectations_store")
store_backend["key_columns"] = store_backend.get(
"key_columns", ["expectation_suite_name"])
super().__init__(store_backend=store_backend,
runtime_environment=runtime_environment)
def remove_key(self, key):
return self.store_backend.remove_key(key)
def serialize(self, key, value):
return self._expectationSuiteSchema.dumps(value,
indent=2,
sort_keys=True)
def deserialize(self, key, value):
return self._expectationSuiteSchema.loads(value)