def test_TupleGCSStoreBackend_base_public_path():
"""
What does this test and why?
the base_public_path parameter allows users to point to a custom DNS when hosting Data docs.
This test will exercise the get_url_for_key method twice to see that we are getting the expected url,
with or without base_public_path
"""
bucket = "leakybucket"
prefix = "this_is_a_test_prefix"
project = "dummy-project"
base_public_path = "http://www.test.com/"
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
mock_bucket = mock_client.get_bucket.return_value
mock_blob = mock_bucket.blob.return_value
my_store_with_base_public_path = TupleGCSStoreBackend(
filepath_template=None,
bucket=bucket,
prefix=prefix,
project=project,
base_public_path=base_public_path,
)
my_store_with_base_public_path.set(("BBB", ),
b"bbb",
content_encoding=None,
content_type="image/png")
run_id = RunIdentifier("my_run_id", datetime.datetime.utcnow())
key = ValidationResultIdentifier(
ExpectationSuiteIdentifier(expectation_suite_name="my_suite_name"),
run_id,
"my_batch_id",
)
run_time_string = run_id.to_tuple()[1]
url = my_store_with_base_public_path.get_public_url_for_key(key.to_tuple())
assert (
url == "http://www.test.com/leakybucket" +
f"/this_is_a_test_prefix/my_suite_name/my_run_id/{run_time_string}/my_batch_id"
)
def test_evaluation_parameter_store_calls_proper_gcs_tuple_store_methods(
mock_parent_list_keys,
mock_gcs_list_keys,
):
"""
What does this test and why?
Demonstrate that EvaluationParameterStore works as expected with TupleGCSStoreBackend
and that the store backend adheres to the Liskov substitution principle.
"""
evaluation_parameter_store = EvaluationParameterStore()
run_id = RunIdentifier()
gcs_store = TupleGCSStoreBackend(bucket="my_bucket", project="my_project")
evaluation_parameter_store._store_backend = gcs_store
# Sanity check to ensure neither parent nor child method has been called
assert not mock_gcs_list_keys.called
assert not mock_parent_list_keys.called
# `get_bind_params` calls the child method due to proper polymorphism
evaluation_parameter_store.get_bind_params(run_id=run_id)
assert mock_gcs_list_keys.called
assert not mock_parent_list_keys.called
def test_TupleGCSStoreBackend():
# pytest.importorskip("google-cloud-storage")
"""
What does this test test and why?
Since no package like moto exists for GCP services, we mock the GCS client
and assert that the store backend makes the right calls for set, get, and list.
TODO : One option may be to have a GCS Store in Docker, which can be use to "actually" run these tests.
"""
bucket = "leakybucket"
prefix = "this_is_a_test_prefix"
project = "dummy-project"
base_public_path = "http://www.test.com/"
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
mock_bucket = mock_client.get_bucket.return_value
mock_blob = mock_bucket.blob.return_value
my_store = TupleGCSStoreBackend(
filepath_template="my_file_{0}",
bucket=bucket,
prefix=prefix,
project=project,
)
my_store.set(("AAA", ), "aaa", content_type="text/html")
mock_gcs_client.assert_called_with("dummy-project")
mock_client.get_bucket.assert_called_with("leakybucket")
mock_bucket.blob.assert_called_with(
"this_is_a_test_prefix/my_file_AAA")
# mock_bucket.blob.assert_any_call("this_is_a_test_prefix/.ge_store_backend_id")
mock_blob.upload_from_string.assert_called_with(
b"aaa", content_type="text/html")
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
mock_bucket = mock_client.get_bucket.return_value
mock_blob = mock_bucket.blob.return_value
my_store_with_no_filepath_template = TupleGCSStoreBackend(
filepath_template=None,
bucket=bucket,
prefix=prefix,
project=project)
my_store_with_no_filepath_template.set(("AAA", ),
b"aaa",
content_encoding=None,
content_type="image/png")
mock_gcs_client.assert_called_with("dummy-project")
mock_client.get_bucket.assert_called_with("leakybucket")
mock_bucket.blob.assert_called_with("this_is_a_test_prefix/AAA")
# mock_bucket.blob.assert_any_call("this_is_a_test_prefix/.ge_store_backend_id")
mock_blob.upload_from_string.assert_called_with(
b"aaa", content_type="image/png")
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
mock_bucket = mock_client.get_bucket.return_value
mock_blob = mock_bucket.get_blob.return_value
mock_str = mock_blob.download_as_string.return_value
my_store.get(("BBB", ))
mock_gcs_client.assert_called_once_with("dummy-project")
mock_client.get_bucket.assert_called_once_with("leakybucket")
mock_bucket.get_blob.assert_called_once_with(
"this_is_a_test_prefix/my_file_BBB")
mock_blob.download_as_string.assert_called_once()
mock_str.decode.assert_called_once_with("utf-8")
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
my_store.list_keys()
mock_client.list_blobs.assert_called_once_with(
"leakybucket", prefix="this_is_a_test_prefix")
my_store.remove_key("leakybucket")
from google.cloud.exceptions import NotFound
try:
mock_client.get_bucket.assert_called_once_with("leakybucket")
except NotFound:
pass
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_gcs_client.side_effect = InvalidKeyError(
"Hi I am an InvalidKeyError")
with pytest.raises(InvalidKeyError):
my_store.get(("non_existent_key", ))
run_id = RunIdentifier("my_run_id", datetime.datetime.utcnow())
key = ValidationResultIdentifier(
ExpectationSuiteIdentifier(expectation_suite_name="my_suite_name"),
run_id,
"my_batch_id",
)
run_time_string = run_id.to_tuple()[1]
url = my_store_with_no_filepath_template.get_url_for_key(key.to_tuple())
assert (
url == "https://storage.googleapis.com/leakybucket" +
f"/this_is_a_test_prefix/my_suite_name/my_run_id/{run_time_string}/my_batch_id"
)
def test_StoreBackend_id_initialization(tmp_path_factory):
"""
What does this test and why?
A StoreBackend should have a store_backend_id property. That store_backend_id should be read and initialized
from an existing persistent store_backend_id during instantiation, or a new store_backend_id should be generated
and persisted. The store_backend_id should be a valid UUIDv4
If a new store_backend_id cannot be persisted, use an ephemeral store_backend_id.
Persistence should be in a .ge_store_backend_id file for for filesystem and blob-stores.
Note: StoreBackend & TupleStoreBackend are abstract classes, so we will test the
concrete classes that inherit from them.
See also test_database_store_backend::test_database_store_backend_id_initialization
"""
# InMemoryStoreBackend
# Initialize without store_backend_id and check that it is generated correctly
in_memory_store_backend = InMemoryStoreBackend()
check_store_backend_store_backend_id_functionality(
store_backend=in_memory_store_backend)
# Create a new store with the same config and make sure it reports the same store_backend_id
# in_memory_store_backend_duplicate = InMemoryStoreBackend()
# assert in_memory_store_backend.store_backend_id == in_memory_store_backend_duplicate.store_backend_id
# This is not currently implemented for the InMemoryStoreBackend, the store_backend_id is ephemeral since
# there is no place to persist it.
# TupleFilesystemStoreBackend
# Initialize without store_backend_id and check that it is generated correctly
path = "dummy_str"
project_path = str(
tmp_path_factory.mktemp("test_StoreBackend_id_initialization__dir"))
tuple_filesystem_store_backend = TupleFilesystemStoreBackend(
root_directory=os.path.abspath(path),
base_directory=project_path,
# filepath_template="my_file_{0}",
)
# Check that store_backend_id is created on instantiation, before being accessed
desired_directory_tree_str = """\
test_StoreBackend_id_initialization__dir0/
.ge_store_backend_id
"""
assert gen_directory_tree_str(project_path) == desired_directory_tree_str
check_store_backend_store_backend_id_functionality(
store_backend=tuple_filesystem_store_backend)
assert gen_directory_tree_str(project_path) == desired_directory_tree_str
# Repeat the above with a filepath template
project_path_with_filepath_template = str(
tmp_path_factory.mktemp("test_StoreBackend_id_initialization__dir"))
tuple_filesystem_store_backend_with_filepath_template = TupleFilesystemStoreBackend(
root_directory=os.path.abspath(path),
base_directory=project_path_with_filepath_template,
filepath_template="my_file_{0}",
)
check_store_backend_store_backend_id_functionality(
store_backend=tuple_filesystem_store_backend_with_filepath_template)
assert (gen_directory_tree_str(project_path_with_filepath_template) == """\
test_StoreBackend_id_initialization__dir1/
.ge_store_backend_id
""")
# Create a new store with the same config and make sure it reports the same store_backend_id
tuple_filesystem_store_backend_duplicate = TupleFilesystemStoreBackend(
root_directory=os.path.abspath(path),
base_directory=project_path,
# filepath_template="my_file_{0}",
)
check_store_backend_store_backend_id_functionality(
store_backend=tuple_filesystem_store_backend_duplicate)
assert (tuple_filesystem_store_backend.store_backend_id ==
tuple_filesystem_store_backend_duplicate.store_backend_id)
# TupleS3StoreBackend
# Initialize without store_backend_id and check that it is generated correctly
bucket = "leakybucket"
prefix = "this_is_a_test_prefix"
# create a bucket in Moto's mock AWS environment
conn = boto3.resource("s3", region_name="us-east-1")
conn.create_bucket(Bucket=bucket)
s3_store_backend = TupleS3StoreBackend(
filepath_template="my_file_{0}",
bucket=bucket,
prefix=prefix,
)
check_store_backend_store_backend_id_functionality(
store_backend=s3_store_backend)
# Create a new store with the same config and make sure it reports the same store_backend_id
s3_store_backend_duplicate = TupleS3StoreBackend(
filepath_template="my_file_{0}",
bucket=bucket,
prefix=prefix,
)
check_store_backend_store_backend_id_functionality(
store_backend=s3_store_backend_duplicate)
assert (s3_store_backend.store_backend_id ==
s3_store_backend_duplicate.store_backend_id)
# TODO: Fix GCS Testing
# TupleGCSStoreBackend
# Initialize without store_backend_id and check that it is generated correctly
bucket = "leakybucket"
prefix = "this_is_a_test_prefix"
project = "dummy-project"
base_public_path = "http://www.test.com/"
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
gcs_store_backend_with_base_public_path = TupleGCSStoreBackend(
filepath_template=None,
bucket=bucket,
prefix=prefix,
project=project,
base_public_path=base_public_path,
)
gcs_store_backend_with_base_public_path_duplicate = TupleGCSStoreBackend(
filepath_template=None,
bucket=bucket,
prefix=prefix,
project=project,
base_public_path=base_public_path,
)
assert gcs_store_backend_with_base_public_path.store_backend_id is not None
# Currently we don't have a good way to mock GCS functionality
# check_store_backend_store_backend_id_functionality(store_backend=gcs_store_backend_with_base_public_path)
# Create a new store with the same config and make sure it reports the same store_backend_id
assert (
gcs_store_backend_with_base_public_path.store_backend_id ==
gcs_store_backend_with_base_public_path_duplicate.store_backend_id)
store_error_uuid = "00000000-0000-0000-0000-00000000e003"
assert (gcs_store_backend_with_base_public_path.store_backend_id !=
store_error_uuid)
assert (
gcs_store_backend_with_base_public_path_duplicate.store_backend_id
!= store_error_uuid)
def test_TupleGCSStoreBackend():
pytest.importorskip("google-cloud-storage")
"""
What does this test test and why?
Since no package like moto exists for GCP services, we mock the GCS client
and assert that the store backend makes the right calls for set, get, and list.
"""
bucket = "leakybucket"
prefix = "this_is_a_test_prefix"
project = "dummy-project"
my_store = TupleGCSStoreBackend(filepath_template="my_file_{0}",
bucket=bucket,
prefix=prefix,
project=project)
my_store_with_no_filepath_template = TupleGCSStoreBackend(
filepath_template=None, bucket=bucket, prefix=prefix, project=project)
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
mock_bucket = mock_client.get_bucket.return_value
mock_blob = mock_bucket.blob.return_value
my_store.set(("AAA", ), "aaa", content_type="text/html")
mock_gcs_client.assert_called_once_with("dummy-project")
mock_client.get_bucket.assert_called_once_with("leakybucket")
mock_bucket.blob.assert_called_once_with(
"this_is_a_test_prefix/my_file_AAA")
mock_blob.upload_from_string.assert_called_once_with(
b"aaa", content_encoding="utf-8", content_type="text/html")
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
mock_bucket = mock_client.get_bucket.return_value
mock_blob = mock_bucket.blob.return_value
my_store_with_no_filepath_template.set(("AAA", ),
b"aaa",
content_encoding=None,
content_type="image/png")
mock_gcs_client.assert_called_once_with("dummy-project")
mock_client.get_bucket.assert_called_once_with("leakybucket")
mock_bucket.blob.assert_called_once_with("this_is_a_test_prefix/AAA")
mock_blob.upload_from_string.assert_called_once_with(
b"aaa", content_type="image/png")
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
mock_bucket = mock_client.get_bucket.return_value
mock_blob = mock_bucket.get_blob.return_value
mock_str = mock_blob.download_as_string.return_value
my_store.get(("BBB", ))
mock_gcs_client.assert_called_once_with("dummy-project")
mock_client.get_bucket.assert_called_once_with("leakybucket")
mock_bucket.get_blob.assert_called_once_with(
"this_is_a_test_prefix/my_file_BBB")
mock_blob.download_as_string.assert_called_once()
mock_str.decode.assert_called_once_with("utf-8")
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
my_store.list_keys()
mock_client.list_blobs.assert_called_once_with(
"leakybucket", prefix="this_is_a_test_prefix")
my_store.remove_key("leakybucket", prefix="this_is_a_test_prefix")
from google.cloud.exceptions import NotFound
try:
mock_client.get_bucket.assert_called_once_with("leakybucket")
except NotFound:
pass
