def test_TupleGCSStoreBackend():
# pytest.importorskip("google-cloud-storage")
"""
What does this test test and why?
Since no package like moto exists for GCP services, we mock the GCS client
and assert that the store backend makes the right calls for set, get, and list.
TODO : One option may be to have a GCS Store in Docker, which can be use to "actually" run these tests.
"""
bucket = "leakybucket"
prefix = "this_is_a_test_prefix"
project = "dummy-project"
base_public_path = "http://www.test.com/"
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
mock_bucket = mock_client.get_bucket.return_value
mock_blob = mock_bucket.blob.return_value
my_store = TupleGCSStoreBackend(
filepath_template="my_file_{0}",
bucket=bucket,
prefix=prefix,
project=project,
)
my_store.set(("AAA", ), "aaa", content_type="text/html")
mock_gcs_client.assert_called_with("dummy-project")
mock_client.get_bucket.assert_called_with("leakybucket")
mock_bucket.blob.assert_called_with(
"this_is_a_test_prefix/my_file_AAA")
# mock_bucket.blob.assert_any_call("this_is_a_test_prefix/.ge_store_backend_id")
mock_blob.upload_from_string.assert_called_with(
b"aaa", content_type="text/html")
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
mock_bucket = mock_client.get_bucket.return_value
mock_blob = mock_bucket.blob.return_value
my_store_with_no_filepath_template = TupleGCSStoreBackend(
filepath_template=None,
bucket=bucket,
prefix=prefix,
project=project)
my_store_with_no_filepath_template.set(("AAA", ),
b"aaa",
content_encoding=None,
content_type="image/png")
mock_gcs_client.assert_called_with("dummy-project")
mock_client.get_bucket.assert_called_with("leakybucket")
mock_bucket.blob.assert_called_with("this_is_a_test_prefix/AAA")
# mock_bucket.blob.assert_any_call("this_is_a_test_prefix/.ge_store_backend_id")
mock_blob.upload_from_string.assert_called_with(
b"aaa", content_type="image/png")
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
mock_bucket = mock_client.get_bucket.return_value
mock_blob = mock_bucket.get_blob.return_value
mock_str = mock_blob.download_as_string.return_value
my_store.get(("BBB", ))
mock_gcs_client.assert_called_once_with("dummy-project")
mock_client.get_bucket.assert_called_once_with("leakybucket")
mock_bucket.get_blob.assert_called_once_with(
"this_is_a_test_prefix/my_file_BBB")
mock_blob.download_as_string.assert_called_once()
mock_str.decode.assert_called_once_with("utf-8")
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
my_store.list_keys()
mock_client.list_blobs.assert_called_once_with(
"leakybucket", prefix="this_is_a_test_prefix")
my_store.remove_key("leakybucket")
from google.cloud.exceptions import NotFound
try:
mock_client.get_bucket.assert_called_once_with("leakybucket")
except NotFound:
pass
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_gcs_client.side_effect = InvalidKeyError(
"Hi I am an InvalidKeyError")
with pytest.raises(InvalidKeyError):
my_store.get(("non_existent_key", ))
run_id = RunIdentifier("my_run_id", datetime.datetime.utcnow())
key = ValidationResultIdentifier(
ExpectationSuiteIdentifier(expectation_suite_name="my_suite_name"),
run_id,
"my_batch_id",
)
run_time_string = run_id.to_tuple()[1]
url = my_store_with_no_filepath_template.get_url_for_key(key.to_tuple())
assert (
url == "https://storage.googleapis.com/leakybucket" +
f"/this_is_a_test_prefix/my_suite_name/my_run_id/{run_time_string}/my_batch_id"
)
def test_TupleGCSStoreBackend():
pytest.importorskip("google-cloud-storage")
"""
What does this test test and why?
Since no package like moto exists for GCP services, we mock the GCS client
and assert that the store backend makes the right calls for set, get, and list.
"""
bucket = "leakybucket"
prefix = "this_is_a_test_prefix"
project = "dummy-project"
my_store = TupleGCSStoreBackend(filepath_template="my_file_{0}",
bucket=bucket,
prefix=prefix,
project=project)
my_store_with_no_filepath_template = TupleGCSStoreBackend(
filepath_template=None, bucket=bucket, prefix=prefix, project=project)
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
mock_bucket = mock_client.get_bucket.return_value
mock_blob = mock_bucket.blob.return_value
my_store.set(("AAA", ), "aaa", content_type="text/html")
mock_gcs_client.assert_called_once_with("dummy-project")
mock_client.get_bucket.assert_called_once_with("leakybucket")
mock_bucket.blob.assert_called_once_with(
"this_is_a_test_prefix/my_file_AAA")
mock_blob.upload_from_string.assert_called_once_with(
b"aaa", content_encoding="utf-8", content_type="text/html")
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
mock_bucket = mock_client.get_bucket.return_value
mock_blob = mock_bucket.blob.return_value
my_store_with_no_filepath_template.set(("AAA", ),
b"aaa",
content_encoding=None,
content_type="image/png")
mock_gcs_client.assert_called_once_with("dummy-project")
mock_client.get_bucket.assert_called_once_with("leakybucket")
mock_bucket.blob.assert_called_once_with("this_is_a_test_prefix/AAA")
mock_blob.upload_from_string.assert_called_once_with(
b"aaa", content_type="image/png")
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
mock_bucket = mock_client.get_bucket.return_value
mock_blob = mock_bucket.get_blob.return_value
mock_str = mock_blob.download_as_string.return_value
my_store.get(("BBB", ))
mock_gcs_client.assert_called_once_with("dummy-project")
mock_client.get_bucket.assert_called_once_with("leakybucket")
mock_bucket.get_blob.assert_called_once_with(
"this_is_a_test_prefix/my_file_BBB")
mock_blob.download_as_string.assert_called_once()
mock_str.decode.assert_called_once_with("utf-8")
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
mock_client = mock_gcs_client.return_value
my_store.list_keys()
mock_client.list_blobs.assert_called_once_with(
"leakybucket", prefix="this_is_a_test_prefix")
my_store.remove_key("leakybucket", prefix="this_is_a_test_prefix")
from google.cloud.exceptions import NotFound
try:
mock_client.get_bucket.assert_called_once_with("leakybucket")
except NotFound:
pass