def test_TupleS3StoreBackend_store_backend_id():
# TupleS3StoreBackend
# Initialize without store_backend_id and check that it is generated correctly
bucket = "leakybucket2"
prefix = "this_is_a_test_prefix"
# create a bucket in Moto's mock AWS environment
conn = boto3.resource("s3", region_name="us-east-1")
conn.create_bucket(Bucket=bucket)
s3_store_backend = TupleS3StoreBackend(
filepath_template="my_file_{0}",
bucket=bucket,
prefix=prefix,
)
check_store_backend_store_backend_id_functionality(
store_backend=s3_store_backend)
# Create a new store with the same config and make sure it reports the same store_backend_id
s3_store_backend_duplicate = TupleS3StoreBackend(
filepath_template="my_file_{0}",
bucket=bucket,
prefix=prefix,
)
store_error_uuid = "00000000-0000-0000-0000-00000000e003"
assert s3_store_backend.store_backend_id != store_error_uuid
assert s3_store_backend_duplicate.store_backend_id != store_error_uuid
assert (s3_store_backend.store_backend_id ==
s3_store_backend_duplicate.store_backend_id)
def test_TupleS3StoreBackend_list_over_1000_keys():
"""
What does this test test and why?
TupleS3StoreBackend.list_keys() should be able to list over 1000 keys
which is the current limit for boto3.list_objects and boto3.list_objects_v2 methods.
See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/paginators.html
We will create a bucket with over 1000 keys, list them with TupleS3StoreBackend.list_keys()
and make sure all are accounted for.
"""
bucket = "leakybucket"
prefix = "my_prefix"
# create a bucket in Moto's mock AWS environment
conn = boto3.resource("s3", region_name="us-east-1")
conn.create_bucket(Bucket=bucket)
# Assert that the bucket is empty before creating store
assert (boto3.client("s3").list_objects_v2(
Bucket=bucket, Prefix=prefix).get("Contents") is None)
my_store = TupleS3StoreBackend(
filepath_template="my_file_{0}",
bucket=bucket,
prefix=prefix,
)
# We should be able to list keys, even when empty
# len(keys) == 1 because of the .ge_store_backend_id
keys = my_store.list_keys()
assert len(keys) == 1
# Add more than 1000 keys
max_keys_in_a_single_call = 1000
num_keys_to_add = int(1.2 * max_keys_in_a_single_call)
for key_num in range(num_keys_to_add):
my_store.set(
(f"AAA_{key_num}", ),
f"aaa_{key_num}",
content_type="text/html; charset=utf-8",
)
assert my_store.get(("AAA_0", )) == "aaa_0"
assert my_store.get(
(f"AAA_{num_keys_to_add-1}", )) == f"aaa_{num_keys_to_add-1}"
# Without pagination only list max_keys_in_a_single_call
# This is belt and suspenders to make sure mocking s3 list_objects_v2 implements
# the same limit as the actual s3 api
assert (len(
boto3.client("s3").list_objects_v2(
Bucket=bucket,
Prefix=prefix)["Contents"]) == max_keys_in_a_single_call)
# With pagination list all keys
keys = my_store.list_keys()
# len(keys) == num_keys_to_add + 1 because of the .ge_store_backend_id
assert len(keys) == num_keys_to_add + 1
def test_TupleS3StoreBackend_with_prefix():
"""
What does this test test and why?
We will exercise the store backend's set method twice and then verify
that the we calling get and list methods will return the expected keys.
We will also check that the objects are stored on S3 at the expected location,
and that the correct S3 URL for the object can be retrieved.
"""
bucket = "leakybucket"
prefix = "this_is_a_test_prefix"
# create a bucket in Moto's mock AWS environment
conn = boto3.resource("s3", region_name="us-east-1")
conn.create_bucket(Bucket=bucket)
my_store = TupleS3StoreBackend(
filepath_template="my_file_{0}",
bucket=bucket,
prefix=prefix,
)
# We should be able to list keys, even when empty
keys = my_store.list_keys()
assert len(keys) == 0
my_store.set(("AAA", ), "aaa", content_type="text/html; charset=utf-8")
assert my_store.get(("AAA", )) == "aaa"
obj = boto3.client("s3").get_object(Bucket=bucket,
Key=prefix + "/my_file_AAA")
assert obj["ContentType"] == "text/html; charset=utf-8"
assert obj["ContentEncoding"] == "utf-8"
my_store.set(("BBB", ), "bbb")
assert my_store.get(("BBB", )) == "bbb"
assert set(my_store.list_keys()) == {("AAA", ), ("BBB", )}
assert set([
s3_object_info["Key"] for s3_object_info in boto3.client(
"s3").list_objects(Bucket=bucket, Prefix=prefix)["Contents"]
]) == {
"this_is_a_test_prefix/my_file_AAA",
"this_is_a_test_prefix/my_file_BBB"
}
assert my_store.get_url_for_key(
("AAA", )) == "https://s3.amazonaws.com/%s/%s/my_file_AAA" % (bucket,
prefix)
assert my_store.get_url_for_key(
("BBB", )) == "https://s3.amazonaws.com/%s/%s/my_file_BBB" % (bucket,
prefix)
my_store.remove_key(("BBB", ))
with pytest.raises(ClientError) as exc:
my_store.get(("BBB", ))
assert exc.value.response["Error"]["Code"] == "NoSuchKey"
def test_TupleS3StoreBackend_with_prefix():
"""
What does this test test and why?
We will exercise the store backend's set method twice and then verify
that the we calling get and list methods will return the expected keys.
We will also check that the objects are stored on S3 at the expected location,
and that the correct S3 URL for the object can be retrieved.
"""
bucket = "leakybucket"
prefix = "this_is_a_test_prefix"
# create a bucket in Moto's mock AWS environment
conn = boto3.resource('s3', region_name='us-east-1')
conn.create_bucket(Bucket=bucket)
my_store = TupleS3StoreBackend(
filepath_template="my_file_{0}",
bucket=bucket,
prefix=prefix,
)
# We should be able to list keys, even when empty
keys = my_store.list_keys()
assert len(keys) == 0
my_store.set(("AAA", ), "aaa", content_type='text/html; charset=utf-8')
assert my_store.get(("AAA", )) == 'aaa'
obj = boto3.client('s3').get_object(Bucket=bucket,
Key=prefix + "/my_file_AAA")
assert obj["ContentType"] == 'text/html; charset=utf-8'
assert obj["ContentEncoding"] == 'utf-8'
my_store.set(("BBB", ), "bbb")
assert my_store.get(("BBB", )) == 'bbb'
assert set(my_store.list_keys()) == {("AAA", ), ("BBB", )}
assert set([
s3_object_info['Key'] for s3_object_info in boto3.client(
's3').list_objects(Bucket=bucket, Prefix=prefix)['Contents']
]) == {
'this_is_a_test_prefix/my_file_AAA',
'this_is_a_test_prefix/my_file_BBB'
}
assert my_store.get_url_for_key(
('AAA', )) == 'https://s3.amazonaws.com/%s/%s/my_file_AAA' % (bucket,
prefix)
assert my_store.get_url_for_key(
('BBB', )) == 'https://s3.amazonaws.com/%s/%s/my_file_BBB' % (bucket,
prefix)
def test_evaluation_parameter_store_calls_proper_cloud_tuple_store_methods(
mock_parent_list_keys,
mock_s3_list_keys,
):
"""
What does this test and why?
Demonstrate that EvaluationParameterStore works as expected with TupleS3StoreBackend
and that the store backend adheres to the Liskov substitution principle.
"""
evaluation_parameter_store = EvaluationParameterStore()
run_id = RunIdentifier()
s3_store = TupleS3StoreBackend(bucket="my_bucket")
evaluation_parameter_store._store_backend = s3_store
# Sanity check to ensure neither parent nor child method has been called
assert not mock_s3_list_keys.called
assert not mock_parent_list_keys.called
# `get_bind_params` calls the child method due to proper polymorphism
evaluation_parameter_store.get_bind_params(run_id=run_id)
assert mock_s3_list_keys.called
assert not mock_parent_list_keys.called
def test_StoreBackend_id_initialization(tmp_path_factory):
"""
What does this test and why?
A StoreBackend should have a store_backend_id property. That store_backend_id should be read and initialized
from an existing persistent store_backend_id during instantiation, or a new store_backend_id should be generated
and persisted. The store_backend_id should be a valid UUIDv4
If a new store_backend_id cannot be persisted, use an ephemeral store_backend_id.
Persistence should be in a .ge_store_backend_id file for for filesystem and blob-stores.
Note: StoreBackend & TupleStoreBackend are abstract classes, so we will test the
concrete classes that inherit from them.
See also test_database_store_backend::test_database_store_backend_id_initialization
"""
# InMemoryStoreBackend
# Initialize without store_backend_id and check that it is generated correctly
in_memory_store_backend = InMemoryStoreBackend()
check_store_backend_store_backend_id_functionality(
store_backend=in_memory_store_backend)
# Create a new store with the same config and make sure it reports the same store_backend_id
# in_memory_store_backend_duplicate = InMemoryStoreBackend()
# assert in_memory_store_backend.store_backend_id == in_memory_store_backend_duplicate.store_backend_id
# This is not currently implemented for the InMemoryStoreBackend, the store_backend_id is ephemeral since
# there is no place to persist it.
# TupleFilesystemStoreBackend
# Initialize without store_backend_id and check that it is generated correctly
path = "dummy_str"
project_path = str(
tmp_path_factory.mktemp("test_StoreBackend_id_initialization__dir"))
tuple_filesystem_store_backend = TupleFilesystemStoreBackend(
root_directory=os.path.abspath(path),
base_directory=project_path,
# filepath_template="my_file_{0}",
)
# Check that store_backend_id is created on instantiation, before being accessed
desired_directory_tree_str = """\
test_StoreBackend_id_initialization__dir0/
.ge_store_backend_id
"""
assert gen_directory_tree_str(project_path) == desired_directory_tree_str
check_store_backend_store_backend_id_functionality(
store_backend=tuple_filesystem_store_backend)
assert gen_directory_tree_str(project_path) == desired_directory_tree_str
# Repeat the above with a filepath template
project_path_with_filepath_template = str(
tmp_path_factory.mktemp("test_StoreBackend_id_initialization__dir"))
tuple_filesystem_store_backend_with_filepath_template = TupleFilesystemStoreBackend(
root_directory=os.path.abspath(path),
base_directory=project_path_with_filepath_template,
filepath_template="my_file_{0}",
)
check_store_backend_store_backend_id_functionality(
store_backend=tuple_filesystem_store_backend_with_filepath_template)
assert (gen_directory_tree_str(project_path_with_filepath_template) == """\
test_StoreBackend_id_initialization__dir1/
.ge_store_backend_id
""")
# Create a new store with the same config and make sure it reports the same store_backend_id
tuple_filesystem_store_backend_duplicate = TupleFilesystemStoreBackend(
root_directory=os.path.abspath(path),
base_directory=project_path,
# filepath_template="my_file_{0}",
)
check_store_backend_store_backend_id_functionality(
store_backend=tuple_filesystem_store_backend_duplicate)
assert (tuple_filesystem_store_backend.store_backend_id ==
tuple_filesystem_store_backend_duplicate.store_backend_id)
# TupleS3StoreBackend
# Initialize without store_backend_id and check that it is generated correctly
bucket = "leakybucket"
prefix = "this_is_a_test_prefix"
# create a bucket in Moto's mock AWS environment
conn = boto3.resource("s3", region_name="us-east-1")
conn.create_bucket(Bucket=bucket)
s3_store_backend = TupleS3StoreBackend(
filepath_template="my_file_{0}",
bucket=bucket,
prefix=prefix,
)
check_store_backend_store_backend_id_functionality(
store_backend=s3_store_backend)
# Create a new store with the same config and make sure it reports the same store_backend_id
s3_store_backend_duplicate = TupleS3StoreBackend(
filepath_template="my_file_{0}",
bucket=bucket,
prefix=prefix,
)
check_store_backend_store_backend_id_functionality(
store_backend=s3_store_backend_duplicate)
assert (s3_store_backend.store_backend_id ==
s3_store_backend_duplicate.store_backend_id)
# TODO: Fix GCS Testing
# TupleGCSStoreBackend
# Initialize without store_backend_id and check that it is generated correctly
bucket = "leakybucket"
prefix = "this_is_a_test_prefix"
project = "dummy-project"
base_public_path = "http://www.test.com/"
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
gcs_store_backend_with_base_public_path = TupleGCSStoreBackend(
filepath_template=None,
bucket=bucket,
prefix=prefix,
project=project,
base_public_path=base_public_path,
)
gcs_store_backend_with_base_public_path_duplicate = TupleGCSStoreBackend(
filepath_template=None,
bucket=bucket,
prefix=prefix,
project=project,
base_public_path=base_public_path,
)
assert gcs_store_backend_with_base_public_path.store_backend_id is not None
# Currently we don't have a good way to mock GCS functionality
# check_store_backend_store_backend_id_functionality(store_backend=gcs_store_backend_with_base_public_path)
# Create a new store with the same config and make sure it reports the same store_backend_id
assert (
gcs_store_backend_with_base_public_path.store_backend_id ==
gcs_store_backend_with_base_public_path_duplicate.store_backend_id)
store_error_uuid = "00000000-0000-0000-0000-00000000e003"
assert (gcs_store_backend_with_base_public_path.store_backend_id !=
store_error_uuid)
assert (
gcs_store_backend_with_base_public_path_duplicate.store_backend_id
!= store_error_uuid)
def test_tuple_s3_store_backend_slash_conditions():
bucket = "my_bucket"
prefix = None
conn = boto3.resource("s3", region_name="us-east-1")
conn.create_bucket(Bucket=bucket)
client = boto3.client("s3")
my_store = TupleS3StoreBackend(
bucket=bucket,
prefix=prefix,
platform_specific_separator=False,
filepath_prefix="foo__",
filepath_suffix="__bar.json",
)
my_store.set(("my_suite", ), '{"foo": "bar"}')
expected_s3_keys = [
"foo__/.ge_store_backend_id", "foo__/my_suite__bar.json"
]
assert [
obj["Key"] for obj in client.list_objects_v2(Bucket=bucket)["Contents"]
] == expected_s3_keys
assert (my_store.get_url_for_key(
("my_suite",
)) == "https://s3.amazonaws.com/my_bucket/foo__/my_suite__bar.json")
client.delete_objects(
Bucket=bucket,
Delete={"Objects": [{
"Key": key
} for key in expected_s3_keys]})
assert len(client.list_objects_v2(Bucket=bucket).get("Contents", [])) == 0
my_store = TupleS3StoreBackend(
bucket=bucket,
prefix=prefix,
platform_specific_separator=False,
)
my_store.set(("my_suite", ), '{"foo": "bar"}')
expected_s3_keys = [".ge_store_backend_id", "my_suite"]
assert [
obj["Key"] for obj in client.list_objects_v2(Bucket=bucket)["Contents"]
] == expected_s3_keys
assert (my_store.get_url_for_key(
("my_suite", )) == "https://s3.amazonaws.com/my_bucket/my_suite")
client.delete_objects(
Bucket=bucket,
Delete={"Objects": [{
"Key": key
} for key in expected_s3_keys]})
assert len(client.list_objects_v2(Bucket=bucket).get("Contents", [])) == 0
my_store = TupleS3StoreBackend(
bucket=bucket,
prefix=prefix,
platform_specific_separator=True,
)
my_store.set(("my_suite", ), '{"foo": "bar"}')
expected_s3_keys = [".ge_store_backend_id", "my_suite"]
assert [
obj["Key"] for obj in client.list_objects_v2(Bucket=bucket)["Contents"]
] == expected_s3_keys
assert (my_store.get_url_for_key(
("my_suite", )) == "https://s3.amazonaws.com/my_bucket/my_suite")
client.delete_objects(
Bucket=bucket,
Delete={"Objects": [{
"Key": key
} for key in expected_s3_keys]})
assert len(client.list_objects_v2(Bucket=bucket).get("Contents", [])) == 0
prefix = "/foo/"
my_store = TupleS3StoreBackend(bucket=bucket,
prefix=prefix,
platform_specific_separator=True)
my_store.set(("my_suite", ), '{"foo": "bar"}')
expected_s3_keys = ["foo/.ge_store_backend_id", "foo/my_suite"]
assert [
obj["Key"] for obj in client.list_objects_v2(Bucket=bucket)["Contents"]
] == expected_s3_keys
assert (my_store.get_url_for_key(
("my_suite", )) == "https://s3.amazonaws.com/my_bucket/foo/my_suite")
client.delete_objects(
Bucket=bucket,
Delete={"Objects": [{
"Key": key
} for key in expected_s3_keys]})
assert len(client.list_objects_v2(Bucket=bucket).get("Contents", [])) == 0
prefix = "foo"
my_store = TupleS3StoreBackend(bucket=bucket,
prefix=prefix,
platform_specific_separator=True)
my_store.set(("my_suite", ), '{"foo": "bar"}')
expected_s3_keys = ["foo/.ge_store_backend_id", "foo/my_suite"]
assert [
obj["Key"] for obj in client.list_objects_v2(Bucket=bucket)["Contents"]
] == expected_s3_keys
assert (my_store.get_url_for_key(
("my_suite", )) == "https://s3.amazonaws.com/my_bucket/foo/my_suite")
client.delete_objects(
Bucket=bucket,
Delete={"Objects": [{
"Key": key
} for key in expected_s3_keys]})
assert len(client.list_objects_v2(Bucket=bucket).get("Contents", [])) == 0
my_store = TupleS3StoreBackend(bucket=bucket,
prefix=prefix,
platform_specific_separator=False)
my_store.set(("my_suite", ), '{"foo": "bar"}')
expected_s3_keys = ["foo/.ge_store_backend_id", "foo/my_suite"]
assert [
obj["Key"] for obj in client.list_objects_v2(Bucket=bucket)["Contents"]
] == expected_s3_keys
assert (my_store.get_url_for_key(
("my_suite", )) == "https://s3.amazonaws.com/my_bucket/foo/my_suite")
client.delete_objects(
Bucket=bucket,
Delete={"Objects": [{
"Key": key
} for key in expected_s3_keys]})
assert len(client.list_objects_v2(Bucket=bucket).get("Contents", [])) == 0
prefix = "foo/"
my_store = TupleS3StoreBackend(bucket=bucket,
prefix=prefix,
platform_specific_separator=True)
my_store.set(("my_suite", ), '{"foo": "bar"}')
expected_s3_keys = ["foo/.ge_store_backend_id", "foo/my_suite"]
assert [
obj["Key"] for obj in client.list_objects_v2(Bucket=bucket)["Contents"]
] == expected_s3_keys
assert (my_store.get_url_for_key(
("my_suite", )) == "https://s3.amazonaws.com/my_bucket/foo/my_suite")
client.delete_objects(
Bucket=bucket,
Delete={"Objects": [{
"Key": key
} for key in expected_s3_keys]})
assert len(client.list_objects_v2(Bucket=bucket).get("Contents", [])) == 0
my_store = TupleS3StoreBackend(bucket=bucket,
prefix=prefix,
platform_specific_separator=False)
my_store.set(("my_suite", ), '{"foo": "bar"}')
expected_s3_keys = ["foo/.ge_store_backend_id", "foo/my_suite"]
assert [
obj["Key"] for obj in client.list_objects_v2(Bucket=bucket)["Contents"]
] == expected_s3_keys
assert (my_store.get_url_for_key(
("my_suite", )) == "https://s3.amazonaws.com/my_bucket/foo/my_suite")
client.delete_objects(
Bucket=bucket,
Delete={"Objects": [{
"Key": key
} for key in expected_s3_keys]})
assert len(client.list_objects_v2(Bucket=bucket).get("Contents", [])) == 0
prefix = "/foo"
my_store = TupleS3StoreBackend(bucket=bucket,
prefix=prefix,
platform_specific_separator=True)
my_store.set(("my_suite", ), '{"foo": "bar"}')
expected_s3_keys = ["foo/.ge_store_backend_id", "foo/my_suite"]
assert [
obj["Key"] for obj in client.list_objects_v2(Bucket=bucket)["Contents"]
] == expected_s3_keys
assert (my_store.get_url_for_key(
("my_suite", )) == "https://s3.amazonaws.com/my_bucket/foo/my_suite")
client.delete_objects(
Bucket=bucket,
Delete={"Objects": [{
"Key": key
} for key in expected_s3_keys]})
assert len(client.list_objects_v2(Bucket=bucket).get("Contents", [])) == 0
my_store = TupleS3StoreBackend(bucket=bucket,
prefix=prefix,
platform_specific_separator=False)
my_store.set(("my_suite", ), '{"foo": "bar"}')
expected_s3_keys = ["foo/.ge_store_backend_id", "foo/my_suite"]
assert [
obj["Key"] for obj in client.list_objects_v2(Bucket=bucket)["Contents"]
] == expected_s3_keys
assert (my_store.get_url_for_key(
("my_suite", )) == "https://s3.amazonaws.com/my_bucket/foo/my_suite")
def test_DataContext_construct_data_context_id_uses_id_of_currently_configured_expectations_store(
):
"""
What does this test and why?
A DataContext should have an id. This ID should come from either:
1. configured expectations store store_backend_id
2. great_expectations.yml
3. new generated id from DataContextConfig
This test verifies that DataContext._construct_data_context_id
uses the store_backend_id from the currently configured expectations store
when instantiating the DataContext
"""
store_backend_id_filename = StoreBackend.STORE_BACKEND_ID_KEY[0]
bucket = "leakybucket"
expectations_store_prefix = "expectations_store_prefix"
validations_store_prefix = "validations_store_prefix"
data_docs_store_prefix = "data_docs_store_prefix"
data_context_prefix = ""
# Create a bucket in Moto's mock AWS environment
conn = boto3.resource("s3", region_name="us-east-1")
conn.create_bucket(Bucket=bucket)
# Create a TupleS3StoreBackend
# Initialize without store_backend_id and check that the store_backend_id is generated correctly
s3_expectations_store_backend = TupleS3StoreBackend(
filepath_template="my_file_{0}",
bucket=bucket,
prefix=expectations_store_prefix,
)
# Make sure store_backend_id is not the error string
store_error_uuid = "00000000-0000-0000-0000-00000000e003"
s3_expectations_store_backend_id = s3_expectations_store_backend.store_backend_id
assert s3_expectations_store_backend_id != store_error_uuid
# Make sure the bucket contents are as expected
bucket_contents_after_creating_expectation_store = list_s3_bucket_contents(
bucket=bucket, prefix=data_context_prefix)
assert bucket_contents_after_creating_expectation_store == {
f"{expectations_store_prefix}/{store_backend_id_filename}"
}
# Make sure the store_backend_id from the file is equal to reading from the property
expectations_store_backend_id_from_s3_file = get_store_backend_id_from_s3(
bucket=bucket,
prefix=expectations_store_prefix,
key=store_backend_id_filename,
)
assert (expectations_store_backend_id_from_s3_file ==
s3_expectations_store_backend_id)
# Create a DataContext (note existing expectations store already set up)
in_code_data_context_project_config = build_in_code_data_context_project_config(
bucket="leakybucket",
expectations_store_prefix=expectations_store_prefix,
validations_store_prefix=validations_store_prefix,
data_docs_store_prefix=data_docs_store_prefix,
)
in_code_data_context = BaseDataContext(
project_config=in_code_data_context_project_config)
bucket_contents_after_instantiating_BaseDataContext = list_s3_bucket_contents(
bucket=bucket, prefix=data_context_prefix)
assert bucket_contents_after_instantiating_BaseDataContext == {
f"{expectations_store_prefix}/{store_backend_id_filename}",
f"{validations_store_prefix}/{store_backend_id_filename}",
}
# Make sure ids are consistent
in_code_data_context_expectations_store_store_backend_id = (
in_code_data_context.stores["expectations_S3_store"].store_backend_id)
in_code_data_context_data_context_id = in_code_data_context.data_context_id
constructed_data_context_id = in_code_data_context._construct_data_context_id(
)
assert (in_code_data_context_expectations_store_store_backend_id ==
in_code_data_context_data_context_id ==
expectations_store_backend_id_from_s3_file ==
s3_expectations_store_backend_id == constructed_data_context_id)