def test_StoreAction():
fake_in_memory_store = ValidationsStore(root_directory=None,
store_backend={
"class_name":
"InMemoryStoreBackend",
})
stores = {"fake_in_memory_store": fake_in_memory_store}
# NOTE: This is a hack meant to last until we implement runtime_configs
class Object(object):
pass
data_context = Object()
data_context.stores = stores
action = StoreAction(
data_context=data_context,
target_store_name="fake_in_memory_store",
)
assert fake_in_memory_store.list_keys() == []
vr_id = "ValidationResultIdentifier.my_db.default_generator.my_table.default_expectations.prod_20190801"
action.run(validation_result_suite_identifier=ValidationResultIdentifier(
from_string=vr_id),
validation_result_suite={},
data_asset=None)
assert len(fake_in_memory_store.list_keys()) == 1
assert fake_in_memory_store.list_keys()[0].to_string(
) == "ValidationResultIdentifier.my_db.default_generator.my_table.default_expectations.prod_20190801"
assert fake_in_memory_store.get(
ValidationResultIdentifier(
from_string=
"ValidationResultIdentifier.my_db.default_generator.my_table.default_expectations.prod_20190801"
)) == {}
def test_ValidationResultIdentifier_to_tuple(expectation_suite_identifier):
validation_result_identifier = ValidationResultIdentifier(
expectation_suite_identifier, "my_run_id", "my_batch_identifier")
assert validation_result_identifier.to_tuple() == ("my", "expectation",
"suite", "name",
"my_run_id",
"my_batch_identifier")
assert validation_result_identifier.to_fixed_length_tuple() == (
"my.expectation.suite.name", "my_run_id", "my_batch_identifier")
validation_result_identifier_no_run_id = ValidationResultIdentifier(
expectation_suite_identifier, None, "my_batch_identifier")
assert validation_result_identifier_no_run_id.to_tuple() == (
"my", "expectation", "suite", "name", "__none__",
"my_batch_identifier")
assert validation_result_identifier_no_run_id.to_fixed_length_tuple() == (
"my.expectation.suite.name", "__none__", "my_batch_identifier")
validation_result_identifier_no_batch_identifier = ValidationResultIdentifier(
expectation_suite_identifier, "my_run_id", None)
assert validation_result_identifier_no_batch_identifier.to_tuple() == (
"my", "expectation", "suite", "name", "my_run_id", "__none__")
assert validation_result_identifier_no_batch_identifier.to_fixed_length_tuple(
) == ("my.expectation.suite.name", "my_run_id", "__none__")
validation_result_identifier_no_run_id_no_batch_identifier = ValidationResultIdentifier(
expectation_suite_identifier, None, None)
assert validation_result_identifier_no_run_id_no_batch_identifier.to_tuple(
) == ("my", "expectation", "suite", "name", "__none__", "__none__")
assert validation_result_identifier_no_run_id_no_batch_identifier.to_fixed_length_tuple(
) == ("my.expectation.suite.name", "__none__", "__none__")
def test_StoreAction():
fake_in_memory_store = ValidationsStore(
store_backend={
"class_name": "InMemoryStoreBackend",
}
)
stores = {"fake_in_memory_store": fake_in_memory_store}
class Object:
ge_cloud_mode = False
data_context = Object()
data_context.stores = stores
action = StoreValidationResultAction(
data_context=data_context,
target_store_name="fake_in_memory_store",
)
assert fake_in_memory_store.list_keys() == []
action.run(
validation_result_suite_identifier=ValidationResultIdentifier(
expectation_suite_identifier=ExpectationSuiteIdentifier(
expectation_suite_name="default_expectations"
),
run_id=RunIdentifier(run_name="prod_20190801"),
batch_identifier="1234",
),
validation_result_suite=ExpectationSuiteValidationResult(
success=False, results=[]
),
data_asset=None,
)
expected_run_id = RunIdentifier(
run_name="prod_20190801", run_time="20190926T134241.000000Z"
)
assert len(fake_in_memory_store.list_keys()) == 1
stored_identifier = fake_in_memory_store.list_keys()[0]
assert stored_identifier.batch_identifier == "1234"
assert (
stored_identifier.expectation_suite_identifier.expectation_suite_name
== "default_expectations"
)
assert stored_identifier.run_id == expected_run_id
assert fake_in_memory_store.get(
ValidationResultIdentifier(
expectation_suite_identifier=ExpectationSuiteIdentifier(
expectation_suite_name="default_expectations"
),
run_id=expected_run_id,
batch_identifier="1234",
)
) == ExpectationSuiteValidationResult(success=False, results=[])
def test_ValidationsStore_with_TupleS3StoreBackend():
bucket = "test_validation_store_bucket"
prefix = "test/prefix"
# create a bucket in Moto's mock AWS environment
conn = boto3.resource("s3", region_name="us-east-1")
conn.create_bucket(Bucket=bucket)
# First, demonstrate that we pick up default configuration including from an S3TupleS3StoreBackend
my_store = ValidationsStore(store_backend={
"class_name": "TupleS3StoreBackend",
"bucket": bucket,
"prefix": prefix,
})
with pytest.raises(TypeError):
my_store.get("not_a_ValidationResultIdentifier")
ns_1 = ValidationResultIdentifier(
expectation_suite_identifier=ExpectationSuiteIdentifier(
expectation_suite_name="asset.quarantine", ),
run_id="20191007T151224.1234Z_prod_100",
batch_identifier="batch_id",
)
my_store.set(ns_1, ExpectationSuiteValidationResult(success=True))
assert my_store.get(ns_1) == ExpectationSuiteValidationResult(
success=True, statistics={}, results=[])
ns_2 = ValidationResultIdentifier(
expectation_suite_identifier=ExpectationSuiteIdentifier(
expectation_suite_name="asset.quarantine", ),
run_id="20191007T151224.1234Z_prod_200",
batch_identifier="batch_id",
)
my_store.set(ns_2, ExpectationSuiteValidationResult(success=False))
assert my_store.get(ns_2) == ExpectationSuiteValidationResult(
success=False, statistics={}, results=[])
# Verify that internals are working as expected, including the default filepath
assert {
s3_object_info["Key"]
for s3_object_info in boto3.client("s3").list_objects_v2(
Bucket=bucket, Prefix=prefix)["Contents"]
} == {
"test/prefix/asset/quarantine/20191007T151224.1234Z_prod_100/20190926T134241.000000Z/batch_id.json",
"test/prefix/asset/quarantine/20191007T151224.1234Z_prod_200/20190926T134241.000000Z/batch_id.json",
}
print(my_store.list_keys())
assert set(my_store.list_keys()) == {
ns_1,
ns_2,
}
def test_ValidationsStore_with_DatabaseStoreBackend(sa):
# Use sqlite so we don't require postgres for this test.
connection_kwargs = {"drivername": "sqlite"}
# First, demonstrate that we pick up default configuration
my_store = ValidationsStore(
store_backend={
"class_name": "DatabaseStoreBackend",
"credentials": connection_kwargs,
}
)
with pytest.raises(TypeError):
my_store.get("not_a_ValidationResultIdentifier")
ns_1 = ValidationResultIdentifier(
expectation_suite_identifier=ExpectationSuiteIdentifier(
expectation_suite_name="asset.quarantine",
),
run_id="20191007T151224.1234Z_prod_100",
batch_identifier="batch_id",
)
my_store.set(ns_1, ExpectationSuiteValidationResult(success=True))
assert my_store.get(ns_1) == ExpectationSuiteValidationResult(
success=True, statistics={}, results=[]
)
ns_2 = ValidationResultIdentifier(
expectation_suite_identifier=ExpectationSuiteIdentifier(
expectation_suite_name="asset.quarantine",
),
run_id="20191007T151224.1234Z_prod_200",
batch_identifier="batch_id",
)
my_store.set(ns_2, ExpectationSuiteValidationResult(success=False))
assert my_store.get(ns_2) == ExpectationSuiteValidationResult(
success=False, statistics={}, results=[]
)
assert set(my_store.list_keys()) == {
ns_1,
ns_2,
}
"""
What does this test and why?
A Store should be able to report it's store_backend_id
which is set when the StoreBackend is instantiated.
"""
# Check that store_backend_id exists can be read
assert my_store.store_backend_id is not None
# Check that store_backend_id is a valid UUID
assert test_utils.validate_uuid4(my_store.store_backend_id)
def validation_result_suite_id():
return ValidationResultIdentifier(
expectation_suite_identifier=ExpectationSuiteIdentifier(
"asset.default"),
run_id=RunIdentifier(run_name="test_100"),
batch_identifier="1234",
)
def _run_actions(
self,
batch,
expectation_suite_identifier,
expectation_suite,
batch_validation_result,
run_id,
):
"""
Runs all actions configured for this operator on the result of validating one
batch against one expectation suite.
If an action fails with an exception, the method does not continue.
:param batch:
:param expectation_suite:
:param batch_validation_result:
:param run_id:
:return: a dictionary: {action name -> result returned by the action}
"""
batch_actions_results = {}
for action in self.action_list:
# NOTE: Eugene: 2019-09-23: log the info about the batch and the expectation suite
logger.debug(
"Processing validation action with name {}".format(action["name"])
)
if isinstance(batch, Validator):
batch_identifier = batch.active_batch_id
else:
batch_identifier = batch.batch_id
validation_result_id = ValidationResultIdentifier(
expectation_suite_identifier=expectation_suite_identifier,
run_id=run_id,
batch_identifier=batch_identifier,
)
try:
action_result = self.actions[action["name"]].run(
validation_result_suite_identifier=validation_result_id,
validation_result_suite=batch_validation_result,
data_asset=batch,
payload=batch_actions_results,
)
# add action_result
batch_actions_results[action["name"]] = (
{} if action_result is None else action_result
)
batch_actions_results[action["name"]]["class"] = action["action"][
"class_name"
]
except Exception as e:
logger.exception(
"Error running action with name {}".format(action["name"])
)
raise e
return batch_actions_results
def test_SlackNotificationAction(data_context):
renderer = {
"module_name": "great_expectations.render.renderer.slack_renderer",
"class_name": "SlackRenderer",
}
slack_webhook = "https://hooks.slack.com/services/test/slack/webhook"
notify_on = "all"
slack_action = SlackNotificationAction(
data_context=data_context,
renderer=renderer,
slack_webhook=slack_webhook,
notify_on=notify_on
)
validation_result_suite = ExpectationSuiteValidationResult(results=[], success=True,
statistics={'evaluated_expectations': 0,
'successful_expectations': 0,
'unsuccessful_expectations': 0,
'success_percent': None},
meta={
'great_expectations.__version__': 'v0.8.0__develop',
'expectation_suite_name': 'asset.default',
'run_id': 'test_100'})
validation_result_suite_id = ValidationResultIdentifier(expectation_suite_identifier=ExpectationSuiteIdentifier(
"asset.default"), run_id="test_100", batch_identifier="1234")
# TODO: improve this test - currently it is verifying a failed call to Slack
assert slack_action.run(
validation_result_suite_identifier=validation_result_suite_id,
validation_result_suite=validation_result_suite,
data_asset=None
) == None
def test_ValidationsStore_with_TupleFileSystemStoreBackend(tmp_path_factory):
path = str(
tmp_path_factory.mktemp(
"test_ValidationResultStore_with_TupleFileSystemStoreBackend__dir")
)
project_path = str(tmp_path_factory.mktemp("my_dir"))
my_store = ValidationsStore(
store_backend={
"module_name": "great_expectations.data_context.store",
"class_name": "TupleFilesystemStoreBackend",
"base_directory": "my_store/",
},
runtime_environment={"root_directory": path},
)
with pytest.raises(TypeError):
my_store.get("not_a_ValidationResultIdentifier")
ns_1 = ValidationResultIdentifier(
expectation_suite_identifier=ExpectationSuiteIdentifier(
"asset.quarantine"),
run_id="prod-100",
batch_identifier="batch_id",
)
my_store.set(ns_1, ExpectationSuiteValidationResult(success=True))
assert my_store.get(ns_1) == ExpectationSuiteValidationResult(
success=True, statistics={}, results=[])
ns_2 = ValidationResultIdentifier.from_tuple((
"asset",
"quarantine",
"prod-20",
datetime.datetime.now(datetime.timezone.utc),
"batch_id",
))
my_store.set(ns_2, ExpectationSuiteValidationResult(success=False))
assert my_store.get(ns_2) == ExpectationSuiteValidationResult(
success=False, statistics={}, results=[])
print(my_store.list_keys())
assert set(my_store.list_keys()) == {
ns_1,
ns_2,
}
print(gen_directory_tree_str(path))
assert (gen_directory_tree_str(path) == """\
test_ValidationResultStore_with_TupleFileSystemStoreBackend__dir0/
my_store/
asset/
quarantine/
prod-100/
20190926T134241.000000Z/
batch_id.json
prod-20/
20190926T134241.000000Z/
batch_id.json
""")
def test_HtmlSiteStore_S3_backend():
bucket = "test_validation_store_bucket"
prefix = "test/prefix"
# create a bucket in Moto's mock AWS environment
conn = boto3.resource("s3", region_name="us-east-1")
conn.create_bucket(Bucket=bucket)
my_store = HtmlSiteStore(store_backend={
"class_name": "TupleS3StoreBackend",
"bucket": bucket,
"prefix": prefix,
})
with pytest.raises(TypeError):
my_store.get("not_a_ValidationResultIdentifier")
ns_1 = SiteSectionIdentifier(
site_section_name="validations",
resource_identifier=ValidationResultIdentifier(
expectation_suite_identifier=ExpectationSuiteIdentifier(
expectation_suite_name="asset.quarantine", ),
run_id="20191007T151224.1234Z_prod_100",
batch_identifier="1234",
),
)
my_store.set(ns_1, "aaa")
ns_2 = SiteSectionIdentifier(
site_section_name="expectations",
resource_identifier=ExpectationSuiteIdentifier(
expectation_suite_name="asset.quarantine", ),
)
my_store.set(ns_2, "bbb")
assert set(my_store.list_keys()) == {
ns_1.resource_identifier,
ns_2.resource_identifier,
}
# This is a special un-store-like method exposed by the HtmlSiteStore
my_store.write_index_page("index_html_string_content")
# Verify that internals are working as expected, including the default filepath
# paths below should include the batch_parameters
assert {
s3_object_info["Key"]
for s3_object_info in boto3.client("s3").list_objects_v2(
Bucket=bucket, Prefix=prefix)["Contents"]
} == {
"test/prefix/index.html",
"test/prefix/expectations/asset/quarantine.html",
"test/prefix/validations/asset/quarantine/20191007T151224.1234Z_prod_100/20190926T134241.000000Z/1234.html",
}
index_content = (boto3.client("s3").get_object(
Bucket=bucket,
Key="test/prefix/index.html")["Body"].read().decode("utf-8"))
assert index_content == "index_html_string_content"
def test_ValidationResultIdentifier__init__entirely_flat():
ValidationResultIdentifier(
"a",
"b",
"c",
"warning",
"testing-12345",
coerce_types=True,
)
def test_ValidationResultIdentifier__init__partially_flat():
ValidationResultIdentifier(coerce_types=True,
**{
"expectation_suite_identifier": {
"data_asset_name": ("a", "b", "c"),
"expectation_suite_name": "hello",
},
"run_id": "testing-12345",
})
def test_ValidationResultIdentifier__init__mostly_nested():
ValidationResultIdentifier(coerce_types=True,
**{
"expectation_suite_identifier": {
"data_asset_name": ("a", "b", "c"),
"expectation_suite_name": "warning",
},
"run_id": "testing-12345",
})
def validation_result_suite_extended_id():
return ValidationResultIdentifier(
expectation_suite_identifier=ExpectationSuiteIdentifier(
"asset.default"),
run_id=RunIdentifier(run_name="test_100",
run_time="Tue May 08 15:14:45 +0800 2012"),
batch_identifier=BatchIdentifier(batch_identifier="1234",
data_asset_name="asset"),
)
def test_ValidationsStore_with_DatabaseStoreBackend(sa):
# Use sqlite so we don't require postgres for this test.
connection_kwargs = {"drivername": "sqlite"}
# First, demonstrate that we pick up default configuration
my_store = ValidationsStore(
store_backend={
"class_name": "DatabaseStoreBackend",
"credentials": connection_kwargs,
}
)
with pytest.raises(TypeError):
my_store.get("not_a_ValidationResultIdentifier")
ns_1 = ValidationResultIdentifier(
expectation_suite_identifier=ExpectationSuiteIdentifier(
expectation_suite_name="asset.quarantine",
),
run_id="20191007T151224.1234Z_prod_100",
batch_identifier="batch_id",
)
my_store.set(ns_1, ExpectationSuiteValidationResult(success=True))
assert my_store.get(ns_1) == ExpectationSuiteValidationResult(
success=True, statistics={}, results=[]
)
ns_2 = ValidationResultIdentifier(
expectation_suite_identifier=ExpectationSuiteIdentifier(
expectation_suite_name="asset.quarantine",
),
run_id="20191007T151224.1234Z_prod_200",
batch_identifier="batch_id",
)
my_store.set(ns_2, ExpectationSuiteValidationResult(success=False))
assert my_store.get(ns_2) == ExpectationSuiteValidationResult(
success=False, statistics={}, results=[]
)
assert set(my_store.list_keys()) == {
ns_1,
ns_2,
}
def _convert_tuple_to_resource_identifier(self, tuple_):
if tuple_[0] == "expectations":
resource_identifier = ExpectationSuiteIdentifier(*tuple_[1])
elif tuple_[0] == "validations":
resource_identifier = ValidationResultIdentifier(*tuple_[1])
else:
raise Exception("unknown section name: " + tuple_[0])
new_identifier = SiteSectionIdentifier(
site_section_name=tuple_[0],
resource_identifier=resource_identifier)
return new_identifier
def test_ValidationResultIdentifier__init__mostly_nested_with_typed_child():
ValidationResultIdentifier(coerce_types=True,
**{
"expectation_suite_identifier": {
"data_asset_name":
DataAssetIdentifier("a", "b", "c"),
"expectation_suite_name":
"quarantine",
},
"run_id": "testing-12345",
})
def ge_validation_result_suite_id() -> ValidationResultIdentifier:
validation_result_suite_id = ValidationResultIdentifier(
expectation_suite_identifier=ExpectationSuiteIdentifier(
"asset.default"),
run_id=RunIdentifier(
run_name="test_100",
run_time=datetime.fromtimestamp(1640701702, tz=timezone.utc),
),
batch_identifier="010ef8c1cd417910b971f4468f024ec5",
)
return validation_result_suite_id
def test_SlackNotificationAction(data_context):
renderer = {
"module_name": "great_expectations.render.renderer.slack_renderer",
"class_name": "SlackRenderer",
}
slack_webhook = "https://hooks.slack.com/services/test/slack/webhook"
notify_on = "all"
slack_action = SlackNotificationAction(data_context=data_context,
renderer=renderer,
slack_webhook=slack_webhook,
notify_on=notify_on)
validation_result_suite = {
'results': [],
'success': True,
'statistics': {
'evaluated_expectations': 0,
'successful_expectations': 0,
'unsuccessful_expectations': 0,
'success_percent': None
},
'meta': {
'great_expectations.__version__': 'v0.8.0__develop',
'data_asset_name': {
'datasource': 'x',
'generator': 'y',
'generator_asset': 'z'
},
'expectation_suite_name': 'default',
'run_id': '2019-09-25T060538.829112Z'
}
}
validation_result_suite_id = ValidationResultIdentifier(
**{
'expectation_suite_identifier': {
'data_asset_name': {
'datasource': 'x',
'generator': 'y',
'generator_asset': 'z'
},
'expectation_suite_name': 'default'
},
'run_id': 'test_100'
})
#TODO: improve this test - currently it is verifying a failed call to Slack
assert slack_action.run(
validation_result_suite_identifier=validation_result_suite_id,
validation_result_suite=validation_result_suite,
data_asset=None) == None
def test_ValidationResultIdentifier__init__nested_except_the_top_layer():
ValidationResultIdentifier(
{
"data_asset_name": {
"datasource": "a",
"generator": "b",
"generator_asset": "c",
},
"expectation_suite_name": "hello",
},
"testing-12345",
coerce_types=True)
def test_resource_key_passes_run_name_filter():
resource_key = ValidationResultIdentifier(
expectation_suite_identifier=ExpectationSuiteIdentifier("test_suite"),
run_id=RunIdentifier(run_name="foofooprofilingfoo"),
batch_identifier="f14c3d2f6e8028c2db0c25edabdb0d61",
)
assert (resource_key_passes_run_name_filter(
resource_key, run_name_filter={"equals": "profiling"}) is False)
assert (resource_key_passes_run_name_filter(
resource_key, run_name_filter={"equals": "foofooprofilingfoo"}) is
True)
assert (resource_key_passes_run_name_filter(
resource_key, run_name_filter={"not_equals": "profiling"}) is True)
assert (resource_key_passes_run_name_filter(
resource_key, run_name_filter={"not_equals": "foofooprofilingfoo"}) is
False)
assert (resource_key_passes_run_name_filter(
resource_key, run_name_filter={"includes": "profiling"}) is True)
assert (resource_key_passes_run_name_filter(
resource_key, run_name_filter={"includes": "foobar"}) is False)
assert (resource_key_passes_run_name_filter(
resource_key, run_name_filter={"not_includes": "foobar"}) is True)
assert (resource_key_passes_run_name_filter(
resource_key, run_name_filter={"not_includes": "profiling"}) is False)
assert (resource_key_passes_run_name_filter(
resource_key,
run_name_filter={"matches_regex": "(foo){2}profiling("
"foo)+"},
) is True)
assert (resource_key_passes_run_name_filter(
resource_key,
run_name_filter={"matches_regex": "(foo){3}profiling("
"foo)+"},
) is False)
with pytest.warns(DeprecationWarning):
assert (resource_key_passes_run_name_filter(
resource_key, run_name_filter={"eq": "profiling"}) is False)
assert (resource_key_passes_run_name_filter(
resource_key, run_name_filter={"eq": "foofooprofilingfoo"}) is
True)
with pytest.warns(DeprecationWarning):
assert (resource_key_passes_run_name_filter(
resource_key, run_name_filter={"ne": "profiling"}) is True)
assert (resource_key_passes_run_name_filter(
resource_key, run_name_filter={"ne": "foofooprofilingfoo"}) is
False)
def test_resource_identifier_to_string():
assert ValidationResultIdentifier(
coerce_types=True,
**{
"expectation_suite_identifier": {
"data_asset_name": {
"datasource": "a",
"generator": "b",
"generator_asset": "c",
},
"expectation_suite_name": "hello",
},
"run_id": "testing-12345",
}).to_string(
) == "ValidationResultIdentifier.a.b.c.hello.testing-12345"
def test_SlackNotificationAction(data_context_parameterized_expectation_suite):
renderer = {
"module_name": "great_expectations.render.renderer.slack_renderer",
"class_name": "SlackRenderer",
}
slack_webhook = "https://hooks.slack.com/services/test/slack/webhook"
notify_on = "all"
slack_action = SlackNotificationAction(
data_context=data_context_parameterized_expectation_suite,
renderer=renderer,
slack_webhook=slack_webhook,
notify_on=notify_on,
)
validation_result_suite = ExpectationSuiteValidationResult(
results=[],
success=True,
statistics={
"evaluated_expectations": 0,
"successful_expectations": 0,
"unsuccessful_expectations": 0,
"success_percent": None,
},
meta={
"great_expectations_version": "v0.8.0__develop",
"expectation_suite_name": "asset.default",
"run_id": "test_100",
},
)
validation_result_suite_id = ValidationResultIdentifier(
expectation_suite_identifier=ExpectationSuiteIdentifier(
"asset.default"),
run_id="test_100",
batch_identifier="1234",
)
# TODO: improve this test - currently it is verifying a failed call to Slack. It returns a "empty" payload
assert slack_action.run(
validation_result_suite_identifier=validation_result_suite_id,
validation_result_suite=validation_result_suite,
data_asset=None,
) == {
"slack_notification_result": None
}
def test_ValidationResultIdentifier__init__totally_nested():
my_id = ValidationResultIdentifier(coerce_types=True,
**{
"expectation_suite_identifier": {
"data_asset_name": {
"datasource": "a",
"generator": "b",
"generator_asset": "c",
},
"expectation_suite_name":
"failure",
},
"run_id": "testing-12345",
})
assert my_id.to_string(
) == "ValidationResultIdentifier.a.b.c.failure.testing-12345"
def test_TupleGCSStoreBackend_base_public_path():
"""
What does this test and why?
the base_public_path parameter allows users to point to a custom DNS when hosting Data docs.
This test will exercise the get_url_for_key method twice to see that we are getting the expected url,
with or without base_public_path
"""
bucket = "leakybucket"
prefix = "this_is_a_test_prefix"
project = "dummy-project"
base_public_path = "http://www.test.com/"
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
mock_bucket = mock_client.get_bucket.return_value
mock_blob = mock_bucket.blob.return_value
my_store_with_base_public_path = TupleGCSStoreBackend(
filepath_template=None,
bucket=bucket,
prefix=prefix,
project=project,
base_public_path=base_public_path,
)
my_store_with_base_public_path.set(("BBB", ),
b"bbb",
content_encoding=None,
content_type="image/png")
run_id = RunIdentifier("my_run_id", datetime.datetime.utcnow())
key = ValidationResultIdentifier(
ExpectationSuiteIdentifier(expectation_suite_name="my_suite_name"),
run_id,
"my_batch_id",
)
run_time_string = run_id.to_tuple()[1]
url = my_store_with_base_public_path.get_public_url_for_key(key.to_tuple())
assert (
url == "http://www.test.com/leakybucket" +
f"/this_is_a_test_prefix/my_suite_name/my_run_id/{run_time_string}/my_batch_id"
)
def test_parse_string_to_data_context_resource_identifier():
assert parse_string_to_data_context_resource_identifier(
"DataAssetIdentifier.A.B.C") == DataAssetIdentifier("A", "B", "C")
assert parse_string_to_data_context_resource_identifier(
"ValidationResultIdentifier.a.b.c.hello.testing-12345"
) == ValidationResultIdentifier(coerce_types=True,
**{
"expectation_suite_identifier": {
"data_asset_name": {
"datasource": "a",
"generator": "b",
"generator_asset": "c",
},
"expectation_suite_name": "hello",
},
"run_id": "testing-12345"
})
def test_MicrosoftTeams_validation_results_with_datadocs():
validation_result_suite = ExpectationSuiteValidationResult(
results=[],
success=True,
statistics={
"evaluated_expectations": 0,
"successful_expectations": 0,
"unsuccessful_expectations": 0,
"success_percent": None,
},
meta={
"great_expectations_version": "v0.8.0__develop",
"expectation_suite_name": "asset.default",
"run_id": "test_100",
},
)
validation_result_suite_identifier = ValidationResultIdentifier(
expectation_suite_identifier=ExpectationSuiteIdentifier(
"asset.default"),
run_id=RunIdentifier(run_name="test_100",
run_time="Tue May 08 15:14:45 +0800 2012"),
batch_identifier=BatchIdentifier(batch_identifier="1234",
data_asset_name="asset"),
)
data_docs_pages = {"local_site": "file:///localsite/index.html"}
rendered_output = MicrosoftTeamsRenderer().render(
validation_result_suite, validation_result_suite_identifier,
data_docs_pages)
expected_output = {
"attachments": [{
"content": {
"$schema":
"http://adaptivecards.io/schemas/adaptive-card.json",
"actions": [{
"title": "Open data docs",
"type": "Action.OpenUrl",
"url": "file:///localsite/index.html",
}],
"body": [
{
"height":
"auto",
"items": [{
"columns": [{
"items": [
{
"size": "large",
"text": "Validation "
"results",
"type": "TextBlock",
"weight": "bolder",
"wrap": True,
},
{
"isSubtle": True,
"spacing": "none",
"text": "May "
"08 "
"2012 "
"07:14:45",
"type": "TextBlock",
"wrap": True,
},
],
"type":
"Column",
"width":
"stretch",
}],
"type":
"ColumnSet",
}],
"separator":
True,
"type":
"Container",
},
{
"height":
"auto",
"items": [
{
"color": "good",
"horizontalAlignment": "left",
"text": "**Batch validation "
"status:** Success "
"!!!",
"type": "TextBlock",
},
{
"horizontalAlignment": "left",
"text": "**Data asset "
"name:** asset",
"type": "TextBlock",
},
{
"horizontalAlignment": "left",
"text": "**Expectation "
"suite name:** "
"asset.default",
"type": "TextBlock",
},
{
"horizontalAlignment": "left",
"text": "**Run name:** "
"test_100",
"type": "TextBlock",
},
{
"horizontalAlignment": "left",
"text": "**Batch ID:** 1234",
"type": "TextBlock",
},
{
"horizontalAlignment":
"left",
"text":
"**Summary:** *0* "
"of *0* "
"expectations were "
"met",
"type":
"TextBlock",
},
],
"separator":
True,
"type":
"Container",
},
],
"type":
"AdaptiveCard",
"version":
"1.0",
},
"contentType":
"application/vnd.microsoft.card.adaptive",
}],
"type":
"message",
}
assert rendered_output == expected_output
def run(
self,
assets_to_validate,
run_id=None,
evaluation_parameters=None,
run_name=None,
run_time=None,
result_format=None,
):
assert not (run_id and run_name) and not (
run_id and run_time
), "Please provide either a run_id or run_name and/or run_time."
if isinstance(run_id, str) and not run_name:
warnings.warn(
"String run_ids will be deprecated in the future. Please provide a run_id of type "
"RunIdentifier(run_name=None, run_time=None), or a dictionary containing run_name "
"and run_time (both optional). Instead of providing a run_id, you may also provide"
"run_name and run_time separately.",
DeprecationWarning,
)
try:
run_time = parse(run_id)
except (ParserError, TypeError):
pass
run_id = RunIdentifier(run_name=run_id, run_time=run_time)
elif isinstance(run_id, dict):
run_id = RunIdentifier(**run_id)
elif not isinstance(run_id, RunIdentifier):
run_id = RunIdentifier(run_name=run_name, run_time=run_time)
run_results = {}
for item in assets_to_validate:
run_result_obj = {}
batch = self._build_batch_from_item(item)
expectation_suite_identifier = ExpectationSuiteIdentifier(
expectation_suite_name=batch._expectation_suite.expectation_suite_name
)
validation_result_id = ValidationResultIdentifier(
batch_identifier=batch.batch_id,
expectation_suite_identifier=expectation_suite_identifier,
run_id=run_id,
)
batch_validation_result = batch.validate(
run_id=run_id,
result_format=result_format if result_format else self.result_format,
evaluation_parameters=evaluation_parameters,
)
run_result_obj["validation_result"] = batch_validation_result
batch_actions_results = self._run_actions(
batch,
expectation_suite_identifier,
batch._expectation_suite,
batch_validation_result,
run_id,
)
run_result_obj["actions_results"] = batch_actions_results
run_results[validation_result_id] = run_result_obj
return ValidationOperatorResult(
run_id=run_id,
run_results=run_results,
validation_operator_config=self.validation_operator_config,
evaluation_parameters=evaluation_parameters,
)
def run(
self,
assets_to_validate,
run_id=None,
base_expectation_suite_name=None,
evaluation_parameters=None,
run_name=None,
run_time=None,
result_format=None,
):
assert not (run_id and run_name) and not (
run_id and run_time
), "Please provide either a run_id or run_name and/or run_time."
if isinstance(run_id, str) and not run_name:
warnings.warn(
"String run_ids will be deprecated in the future. Please provide a run_id of type "
"RunIdentifier(run_name=None, run_time=None), or a dictionary containing run_name "
"and run_time (both optional). Instead of providing a run_id, you may also provide"
"run_name and run_time separately.",
DeprecationWarning,
)
try:
run_time = parse(run_id)
except (ParserError, TypeError):
pass
run_id = RunIdentifier(run_name=run_id, run_time=run_time)
elif isinstance(run_id, dict):
run_id = RunIdentifier(**run_id)
elif not isinstance(run_id, RunIdentifier):
run_id = RunIdentifier(run_name=run_name, run_time=run_time)
if base_expectation_suite_name is None:
if self.base_expectation_suite_name is None:
raise ValueError(
"base_expectation_suite_name must be configured in the validation operator or passed at runtime"
)
base_expectation_suite_name = self.base_expectation_suite_name
run_results = {}
for item in assets_to_validate:
batch = self._build_batch_from_item(item)
batch_id = batch.batch_id
run_id = run_id
assert not batch_id is None
assert not run_id is None
failure_expectation_suite_identifier = ExpectationSuiteIdentifier(
expectation_suite_name=base_expectation_suite_name
+ self.expectation_suite_name_suffixes[0]
)
failure_validation_result_id = ValidationResultIdentifier(
expectation_suite_identifier=failure_expectation_suite_identifier,
run_id=run_id,
batch_identifier=batch_id,
)
failure_expectation_suite = None
try:
failure_expectation_suite = self.data_context.stores[
self.data_context.expectations_store_name
].get(failure_expectation_suite_identifier)
# NOTE : Abe 2019/09/17 : I'm concerned that this may be too permissive, since
# it will catch any error in the Store, not just KeyErrors. In the longer term, a better
# solution will be to have the Stores catch other known errors and raise KeyErrors,
# so that methods like this can catch and handle a single error type.
except Exception:
logger.debug(
"Failure expectation suite not found: {}".format(
failure_expectation_suite_identifier
)
)
if failure_expectation_suite:
failure_run_result_obj = {"expectation_suite_severity_level": "failure"}
failure_validation_result = batch.validate(
failure_expectation_suite,
result_format=result_format
if result_format
else self.result_format,
evaluation_parameters=evaluation_parameters,
)
failure_run_result_obj["validation_result"] = failure_validation_result
failure_actions_results = self._run_actions(
batch,
failure_expectation_suite_identifier,
failure_expectation_suite,
failure_validation_result,
run_id,
)
failure_run_result_obj["actions_results"] = failure_actions_results
run_results[failure_validation_result_id] = failure_run_result_obj
if not failure_validation_result.success and self.stop_on_first_error:
break
warning_expectation_suite_identifier = ExpectationSuiteIdentifier(
expectation_suite_name=base_expectation_suite_name
+ self.expectation_suite_name_suffixes[1]
)
warning_validation_result_id = ValidationResultIdentifier(
expectation_suite_identifier=warning_expectation_suite_identifier,
run_id=run_id,
batch_identifier=batch.batch_id,
)
warning_expectation_suite = None
try:
warning_expectation_suite = self.data_context.stores[
self.data_context.expectations_store_name
].get(warning_expectation_suite_identifier)
except Exception:
logger.debug(
"Warning expectation suite not found: {}".format(
warning_expectation_suite_identifier
)
)
if warning_expectation_suite:
warning_run_result_obj = {"expectation_suite_severity_level": "warning"}
warning_validation_result = batch.validate(
warning_expectation_suite,
result_format=result_format
if result_format
else self.result_format,
evaluation_parameters=evaluation_parameters,
)
warning_run_result_obj["validation_result"] = warning_validation_result
warning_actions_results = self._run_actions(
batch,
warning_expectation_suite_identifier,
warning_expectation_suite,
warning_validation_result,
run_id,
)
warning_run_result_obj["actions_results"] = warning_actions_results
run_results[warning_validation_result_id] = warning_run_result_obj
validation_operator_result = ValidationOperatorResult(
run_id=run_id,
run_results=run_results,
validation_operator_config=self.validation_operator_config,
evaluation_parameters=evaluation_parameters,
success=all(
[
run_result_obj["validation_result"].success
for run_result_obj in run_results.values()
]
),
)
if self.slack_webhook:
if (
self.notify_on == "all"
or self.notify_on == "success"
and validation_operator_result.success
or self.notify_on == "failure"
and not validation_operator_result.success
):
slack_query = self._build_slack_query(
validation_operator_result=validation_operator_result
)
send_slack_notification(
query=slack_query, slack_webhook=self.slack_webhook
)
return validation_operator_result
def test_TupleGCSStoreBackend():
# pytest.importorskip("google-cloud-storage")
"""
What does this test test and why?
Since no package like moto exists for GCP services, we mock the GCS client
and assert that the store backend makes the right calls for set, get, and list.
TODO : One option may be to have a GCS Store in Docker, which can be use to "actually" run these tests.
"""
bucket = "leakybucket"
prefix = "this_is_a_test_prefix"
project = "dummy-project"
base_public_path = "http://www.test.com/"
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
mock_bucket = mock_client.get_bucket.return_value
mock_blob = mock_bucket.blob.return_value
my_store = TupleGCSStoreBackend(
filepath_template="my_file_{0}",
bucket=bucket,
prefix=prefix,
project=project,
)
my_store.set(("AAA", ), "aaa", content_type="text/html")
mock_gcs_client.assert_called_with("dummy-project")
mock_client.get_bucket.assert_called_with("leakybucket")
mock_bucket.blob.assert_called_with(
"this_is_a_test_prefix/my_file_AAA")
# mock_bucket.blob.assert_any_call("this_is_a_test_prefix/.ge_store_backend_id")
mock_blob.upload_from_string.assert_called_with(
b"aaa", content_type="text/html")
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
mock_bucket = mock_client.get_bucket.return_value
mock_blob = mock_bucket.blob.return_value
my_store_with_no_filepath_template = TupleGCSStoreBackend(
filepath_template=None,
bucket=bucket,
prefix=prefix,
project=project)
my_store_with_no_filepath_template.set(("AAA", ),
b"aaa",
content_encoding=None,
content_type="image/png")
mock_gcs_client.assert_called_with("dummy-project")
mock_client.get_bucket.assert_called_with("leakybucket")
mock_bucket.blob.assert_called_with("this_is_a_test_prefix/AAA")
# mock_bucket.blob.assert_any_call("this_is_a_test_prefix/.ge_store_backend_id")
mock_blob.upload_from_string.assert_called_with(
b"aaa", content_type="image/png")
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
mock_bucket = mock_client.get_bucket.return_value
mock_blob = mock_bucket.get_blob.return_value
mock_str = mock_blob.download_as_string.return_value
my_store.get(("BBB", ))
mock_gcs_client.assert_called_once_with("dummy-project")
mock_client.get_bucket.assert_called_once_with("leakybucket")
mock_bucket.get_blob.assert_called_once_with(
"this_is_a_test_prefix/my_file_BBB")
mock_blob.download_as_string.assert_called_once()
mock_str.decode.assert_called_once_with("utf-8")
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
my_store.list_keys()
mock_client.list_blobs.assert_called_once_with(
"leakybucket", prefix="this_is_a_test_prefix")
my_store.remove_key("leakybucket")
from google.cloud.exceptions import NotFound
try:
mock_client.get_bucket.assert_called_once_with("leakybucket")
except NotFound:
pass
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_gcs_client.side_effect = InvalidKeyError(
"Hi I am an InvalidKeyError")
with pytest.raises(InvalidKeyError):
my_store.get(("non_existent_key", ))
run_id = RunIdentifier("my_run_id", datetime.datetime.utcnow())
key = ValidationResultIdentifier(
ExpectationSuiteIdentifier(expectation_suite_name="my_suite_name"),
run_id,
"my_batch_id",
)
run_time_string = run_id.to_tuple()[1]
url = my_store_with_no_filepath_template.get_url_for_key(key.to_tuple())
assert (
url == "https://storage.googleapis.com/leakybucket" +
f"/this_is_a_test_prefix/my_suite_name/my_run_id/{run_time_string}/my_batch_id"
)