def test_FilesystemStoreBackend_two_way_string_conversion(tmp_path_factory):
path = str(
tmp_path_factory.mktemp(
"test_FilesystemStoreBackend_two_way_string_conversion__dir"
)
)
project_path = str(tmp_path_factory.mktemp("my_dir"))
my_store = TupleFilesystemStoreBackend(
root_directory=os.path.abspath(path),
base_directory=project_path,
filepath_template="{0}/{1}/{2}/foo-{2}-expectations.txt",
)
tuple_ = ("A__a", "B-b", "C")
converted_string = my_store._convert_key_to_filepath(tuple_)
assert converted_string == "A__a/B-b/C/foo-C-expectations.txt"
recovered_key = my_store._convert_filepath_to_key(
"A__a/B-b/C/foo-C-expectations.txt"
)
assert recovered_key == tuple_
with pytest.raises(ValueError):
tuple_ = ("A/a", "B-b", "C")
converted_string = my_store._convert_key_to_filepath(tuple_)
def test_tuple_filesystem_store_filepath_prefix_error(tmp_path_factory):
path = str(tmp_path_factory.mktemp('test_tuple_filesystem_store_filepath_prefix_error'))
project_path = str(tmp_path_factory.mktemp('my_dir'))
with pytest.raises(StoreBackendError) as e:
TupleFilesystemStoreBackend(
root_directory=os.path.abspath(path),
base_directory=project_path,
filepath_prefix="invalid_prefix_ends_with/")
assert "filepath_prefix may not end with" in e.value.message
with pytest.raises(StoreBackendError) as e:
TupleFilesystemStoreBackend(
root_directory=os.path.abspath(path),
base_directory=project_path,
filepath_prefix="invalid_prefix_ends_with\\")
assert "filepath_prefix may not end with" in e.value.message
def test_TupleFilesystemStoreBackend_ignores_jupyter_notebook_checkpoints(
tmp_path_factory):
project_path = str(tmp_path_factory.mktemp('things'))
checkpoint_dir = os.path.join(project_path, ".ipynb_checkpoints")
os.mkdir(checkpoint_dir)
assert os.path.isdir(checkpoint_dir)
nb_file = os.path.join(checkpoint_dir, "foo.json")
with open(nb_file, "w") as f:
f.write("")
assert os.path.isfile(nb_file)
my_store = TupleFilesystemStoreBackend(
root_directory=os.path.abspath("dummy_str"),
base_directory=project_path,
)
my_store.set(("AAA", ), "aaa")
assert my_store.get(("AAA", )) == "aaa"
assert gen_directory_tree_str(project_path) == """\
things0/
AAA
.ipynb_checkpoints/
foo.json
"""
print(my_store.list_keys())
assert set(my_store.list_keys()) == {("AAA", )}
def test_StoreBackend_id_initialization(tmp_path_factory):
"""
What does this test and why?
A StoreBackend should have a store_backend_id property. That store_backend_id should be read and initialized
from an existing persistent store_backend_id during instantiation, or a new store_backend_id should be generated
and persisted. The store_backend_id should be a valid UUIDv4
If a new store_backend_id cannot be persisted, use an ephemeral store_backend_id.
Persistence should be in a .ge_store_backend_id file for for filesystem and blob-stores.
Note: StoreBackend & TupleStoreBackend are abstract classes, so we will test the
concrete classes that inherit from them.
See also test_database_store_backend::test_database_store_backend_id_initialization
"""
# InMemoryStoreBackend
# Initialize without store_backend_id and check that it is generated correctly
in_memory_store_backend = InMemoryStoreBackend()
check_store_backend_store_backend_id_functionality(
store_backend=in_memory_store_backend)
# Create a new store with the same config and make sure it reports the same store_backend_id
# in_memory_store_backend_duplicate = InMemoryStoreBackend()
# assert in_memory_store_backend.store_backend_id == in_memory_store_backend_duplicate.store_backend_id
# This is not currently implemented for the InMemoryStoreBackend, the store_backend_id is ephemeral since
# there is no place to persist it.
# TupleFilesystemStoreBackend
# Initialize without store_backend_id and check that it is generated correctly
path = "dummy_str"
project_path = str(
tmp_path_factory.mktemp("test_StoreBackend_id_initialization__dir"))
tuple_filesystem_store_backend = TupleFilesystemStoreBackend(
root_directory=os.path.abspath(path),
base_directory=project_path,
# filepath_template="my_file_{0}",
)
# Check that store_backend_id is created on instantiation, before being accessed
desired_directory_tree_str = """\
test_StoreBackend_id_initialization__dir0/
.ge_store_backend_id
"""
assert gen_directory_tree_str(project_path) == desired_directory_tree_str
check_store_backend_store_backend_id_functionality(
store_backend=tuple_filesystem_store_backend)
assert gen_directory_tree_str(project_path) == desired_directory_tree_str
# Repeat the above with a filepath template
project_path_with_filepath_template = str(
tmp_path_factory.mktemp("test_StoreBackend_id_initialization__dir"))
tuple_filesystem_store_backend_with_filepath_template = TupleFilesystemStoreBackend(
root_directory=os.path.abspath(path),
base_directory=project_path_with_filepath_template,
filepath_template="my_file_{0}",
)
check_store_backend_store_backend_id_functionality(
store_backend=tuple_filesystem_store_backend_with_filepath_template)
assert (gen_directory_tree_str(project_path_with_filepath_template) == """\
test_StoreBackend_id_initialization__dir1/
.ge_store_backend_id
""")
# Create a new store with the same config and make sure it reports the same store_backend_id
tuple_filesystem_store_backend_duplicate = TupleFilesystemStoreBackend(
root_directory=os.path.abspath(path),
base_directory=project_path,
# filepath_template="my_file_{0}",
)
check_store_backend_store_backend_id_functionality(
store_backend=tuple_filesystem_store_backend_duplicate)
assert (tuple_filesystem_store_backend.store_backend_id ==
tuple_filesystem_store_backend_duplicate.store_backend_id)
# TupleS3StoreBackend
# Initialize without store_backend_id and check that it is generated correctly
bucket = "leakybucket"
prefix = "this_is_a_test_prefix"
# create a bucket in Moto's mock AWS environment
conn = boto3.resource("s3", region_name="us-east-1")
conn.create_bucket(Bucket=bucket)
s3_store_backend = TupleS3StoreBackend(
filepath_template="my_file_{0}",
bucket=bucket,
prefix=prefix,
)
check_store_backend_store_backend_id_functionality(
store_backend=s3_store_backend)
# Create a new store with the same config and make sure it reports the same store_backend_id
s3_store_backend_duplicate = TupleS3StoreBackend(
filepath_template="my_file_{0}",
bucket=bucket,
prefix=prefix,
)
check_store_backend_store_backend_id_functionality(
store_backend=s3_store_backend_duplicate)
assert (s3_store_backend.store_backend_id ==
s3_store_backend_duplicate.store_backend_id)
# TODO: Fix GCS Testing
# TupleGCSStoreBackend
# Initialize without store_backend_id and check that it is generated correctly
bucket = "leakybucket"
prefix = "this_is_a_test_prefix"
project = "dummy-project"
base_public_path = "http://www.test.com/"
with patch("google.cloud.storage.Client",
autospec=True) as mock_gcs_client:
gcs_store_backend_with_base_public_path = TupleGCSStoreBackend(
filepath_template=None,
bucket=bucket,
prefix=prefix,
project=project,
base_public_path=base_public_path,
)
gcs_store_backend_with_base_public_path_duplicate = TupleGCSStoreBackend(
filepath_template=None,
bucket=bucket,
prefix=prefix,
project=project,
base_public_path=base_public_path,
)
assert gcs_store_backend_with_base_public_path.store_backend_id is not None
# Currently we don't have a good way to mock GCS functionality
# check_store_backend_store_backend_id_functionality(store_backend=gcs_store_backend_with_base_public_path)
# Create a new store with the same config and make sure it reports the same store_backend_id
assert (
gcs_store_backend_with_base_public_path.store_backend_id ==
gcs_store_backend_with_base_public_path_duplicate.store_backend_id)
store_error_uuid = "00000000-0000-0000-0000-00000000e003"
assert (gcs_store_backend_with_base_public_path.store_backend_id !=
store_error_uuid)
assert (
gcs_store_backend_with_base_public_path_duplicate.store_backend_id
!= store_error_uuid)
def test_TupleFilesystemStoreBackend(tmp_path_factory):
path = "dummy_str"
project_path = str(
tmp_path_factory.mktemp("test_TupleFilesystemStoreBackend__dir"))
base_public_path = "http://www.test.com/"
my_store = TupleFilesystemStoreBackend(
root_directory=os.path.abspath(path),
base_directory=project_path,
filepath_template="my_file_{0}",
)
with pytest.raises(InvalidKeyError):
my_store.get(("AAA", ))
my_store.set(("AAA", ), "aaa")
assert my_store.get(("AAA", )) == "aaa"
my_store.set(("BBB", ), "bbb")
assert my_store.get(("BBB", )) == "bbb"
assert set(my_store.list_keys()) == {(".ge_store_backend_id", ), ("AAA", ),
("BBB", )}
assert (gen_directory_tree_str(project_path) == """\
test_TupleFilesystemStoreBackend__dir0/
.ge_store_backend_id
my_file_AAA
my_file_BBB
""")
my_store.remove_key(("BBB", ))
with pytest.raises(InvalidKeyError):
assert my_store.get(("BBB", )) == ""
my_store_with_base_public_path = TupleFilesystemStoreBackend(
root_directory=os.path.abspath(path),
base_directory=project_path,
filepath_template="my_file_{0}",
base_public_path=base_public_path,
)
my_store_with_base_public_path.set(("CCC", ), "ccc")
url = my_store_with_base_public_path.get_public_url_for_key(("CCC", ))
assert url == "http://www.test.com/my_file_CCC"
def test_TupleFilesystemStoreBackend(tmp_path_factory):
path = "dummy_str"
project_path = str(
tmp_path_factory.mktemp("test_TupleFilesystemStoreBackend__dir"))
my_store = TupleFilesystemStoreBackend(
root_directory=os.path.abspath(path),
base_directory=project_path,
filepath_template="my_file_{0}",
)
# OPPORTUNITY: potentially standardize error instead of allowing each StoreBackend to raise its own error types
with pytest.raises(FileNotFoundError):
my_store.get(("AAA", ))
my_store.set(("AAA", ), "aaa")
assert my_store.get(("AAA", )) == "aaa"
my_store.set(("BBB", ), "bbb")
assert my_store.get(("BBB", )) == "bbb"
assert set(my_store.list_keys()) == {("AAA", ), ("BBB", )}
assert (gen_directory_tree_str(project_path) == """\
test_TupleFilesystemStoreBackend__dir0/
my_file_AAA
my_file_BBB
""")
my_store.remove_key(("BBB", ))
with pytest.raises(FileNotFoundError):
assert my_store.get(("BBB", )) == ""
def test_TupleFilesystemStoreBackend(tmp_path_factory):
path = "dummy_str"
project_path = str(tmp_path_factory.mktemp("test_TupleFilesystemStoreBackend__dir"))
my_store = TupleFilesystemStoreBackend(
root_directory=os.path.abspath(path),
base_directory=project_path,
filepath_template="my_file_{0}",
)
with pytest.raises(InvalidKeyError):
my_store.get(("AAA",))
my_store.set(("AAA",), "aaa")
assert my_store.get(("AAA",)) == "aaa"
my_store.set(("BBB",), "bbb")
assert my_store.get(("BBB",)) == "bbb"
assert set(my_store.list_keys()) == {("AAA",), ("BBB",)}
assert (
gen_directory_tree_str(project_path)
== """\
test_TupleFilesystemStoreBackend__dir0/
my_file_AAA
my_file_BBB
"""
)
my_store.remove_key(("BBB",))
with pytest.raises(InvalidKeyError):
assert my_store.get(("BBB",)) == ""