async def test_namespace(Adapter, DictDatastore, encode_fn):
k1 = datastore.Key('/c/d')
k2 = datastore.Key('/a/b')
k3 = datastore.Key('/a/b/c/d')
ds = DictDatastore()
async with Adapter(k2, ds) as nd:
await ds.put(k1, encode_fn('cd'))
await ds.put(k3, encode_fn('abcd'))
assert await ds.get_all(k1) == encode_fn('cd')
assert not await ds.contains(k2)
assert await ds.get_all(k3) == encode_fn('abcd')
assert await nd.get_all(k1) == encode_fn('abcd')
assert not await nd.contains(k2)
assert not await nd.contains(k3)
async def test(key, val):
await nd.put(key, val)
assert await nd.get_all(key) == val
assert not await ds.contains(key)
assert not await nd.contains(k2.child(key))
assert await ds.get_all(k2.child(key)) == val
for i in range(0, 10):
await test(datastore.Key(str(i)), encode_fn(f"val{i}"))
async def test_lowercase_key(Adapter, DictDatastore, encode_fn):
ds = DictDatastore()
async with Adapter(ds) as lds:
k1 = datastore.Key('hello')
k2 = datastore.Key('HELLO')
k3 = datastore.Key('HeLlo')
await ds.put(k1, encode_fn('world'))
await ds.put(k2, encode_fn('WORLD'))
assert await ds.get_all(k1) == encode_fn('world')
assert await ds.get_all(k2) == encode_fn('WORLD')
assert not await ds.contains(k3)
assert await lds.get_all(k1) == encode_fn('world')
assert await lds.get_all(k2) == encode_fn('world')
assert await lds.get_all(k3) == encode_fn('world')
async def test(key, val):
await lds.put(key, val)
assert await lds.get_all(k1) == val
assert await lds.get_all(k2) == val
assert await lds.get_all(k3) == val
await test(k1, encode_fn('a'))
await test(k2, encode_fn('b'))
await test(k3, encode_fn('c'))
async def test_stats_tracking(temp_path):
async with FileSystemDatastore.create(temp_path, stats=True) as fs:
assert fs.datastore_stats().size == 0
assert fs.datastore_stats().size_accuracy == "exact"
# Write to datastore key
await fs.put(datastore.Key("/a"), b"1234")
assert fs.datastore_stats().size == 4
# Overwrite datastore key
await fs.put(datastore.Key("/a"), b"Hallo Welt")
assert fs.datastore_stats().size == 10
# Create new datastore key
key_new = await fs.put_new(datastore.Key("/"), b"TC")
assert fs.datastore_stats().size == 12
# Remove datastore keys
await fs.delete(datastore.Key("/a"))
assert fs.datastore_stats().size == 2
await fs.delete(key_new)
assert fs.datastore_stats().size == 0
async def test_nested_path_3_2(Adapter, DictDatastore, encode_fn):
opts = {}
opts['k1'] = datastore.Key('/abcdefghijk')
opts['k2'] = datastore.Key('/abcdefghijki')
opts['k3'] = datastore.Key('/ab/cd/ef/abcdefghijk')
opts['k4'] = datastore.Key('/ab/cd/ef/abcdefghijki')
opts['depth'] = 3
opts['length'] = 2
await subtest_nested_path_ds(Adapter, DictDatastore, encode_fn, **opts)
async def test_namespace_simple(DatastoreTests, Adapter, DictDatastore,
encode_fn):
async with contextlib.AsyncExitStack() as stack:
s1 = stack.push_async_exit(Adapter(datastore.Key('a'),
DictDatastore()))
s2 = stack.push_async_exit(Adapter(datastore.Key('b'),
DictDatastore()))
s3 = stack.push_async_exit(Adapter(datastore.Key('c'),
DictDatastore()))
stores = [s1, s2, s3]
await DatastoreTests(stores, test_put_new=True).subtest_simple()
async def test_stats_restore(temp_path):
async with FileSystemDatastore.create(temp_path, stats=True) as fs:
assert fs.datastore_stats().size == 0
assert fs.datastore_stats().size_accuracy == "exact"
await fs.put(datastore.Key("/a"), b"1234")
assert fs.datastore_stats().size == 4
# Re-open datastore and check that the stats are still there
async with FileSystemDatastore.create(temp_path, stats=True) as fs:
assert fs.datastore_stats().size == 4
assert fs.datastore_stats().size_accuracy == "exact"
# Replace content with stuff written by a non-cooperating datastore user
await (trio.Path(temp_path) / "diskUsage.data").write_text(
json.dumps({
"diskUsage": 3,
"accuracy": "initial-exact"
}),
encoding="utf-8")
# Re-open datastore and check that the non-cooperating stats data is
# properly merged
async with FileSystemDatastore.create(temp_path, stats=True) as fs:
assert fs.datastore_stats().size == 7
assert fs.datastore_stats().size_accuracy == "exact"
def _find_mountpoint(self, key: datastore.Key) \
-> typing.Tuple[typing.Optional[DS], datastore.Key, datastore.Key]:
current = self.mounts
ds = None
offset = 0
for idx, part in enumerate(map(str, key.list)):
mount = current.get(part)
if mount is None:
break
if mount[1] is not None:
ds = mount[1]
offset = idx
current = mount[0]
return ds, datastore.Key(key.list[:offset]), datastore.Key(
key.list[(offset + 1):])
async def create(
cls,
root: typing.Union[os_PathLike_str, str],
*,
case_sensitive: bool = True,
remove_empty: bool = True,
stats: bool = False,
stats_key: datastore.Key = DEFAULT_STATS_KEY
) -> 'FileSystemDatastore':
"""Initialize the datastore with given root directory `root`.
Arguments
---------
root
A path at which to mount this filesystem datastore.
case_sensitive
Keep case of all path items (True) or lower case them (False)
before passing them to the OS. Note that, if the underlying
file system is case-insensitive this option will not prevent
conflicts between paths that differ in case only.
remove_empty
Attempt to remove empty directories in the underlying file
system. While this is enabled every successful delete operation
will be followed by at least one extra context switch to invoke
the `rmdir` system call.
stats
Track summary statistics about the contents of the datastore,
currently only the total apparent size of all files on the disk is
tracked
stats_key
The key/filepath at which to persist statistics between runs
"""
if not root:
raise ValueError(
'root path must not be empty (use \'.\' for current directory)'
)
# Ensure target directory exists
await trio.Path(root).mkdir(parents=True, exist_ok=True)
# Create instance
self = cls(_create_call=True)
# Do the usual constructor stuff
self.root_path = pathlib.PurePath(root)
self.case_sensitive = bool(case_sensitive)
self.remove_empty = bool(remove_empty)
self.stats_key = datastore.Key(stats_key)
# Enable stats processing
if stats:
self._stats_lock = trio.Lock()
await self._init_stats()
return self
async def test_keytransform_reverse_transform(Adapter, DictDatastore,
encode_fn):
def transform(key):
return key.reverse
ds = DictDatastore()
async with Adapter(ds, key_transform=transform) as kt:
k1 = datastore.Key('/a/b/c')
k2 = datastore.Key('/c/b/a')
assert not await ds.contains(k1)
assert not await ds.contains(k2)
assert not await kt.contains(k1)
assert not await kt.contains(k2)
await ds.put(k1, encode_fn('abc'))
assert await ds.get_all(k1) == encode_fn('abc')
assert not await ds.contains(k2)
assert not await kt.contains(k1)
with pytest.raises(KeyError):
await (await kt.get(k1)).aclose()
assert await (await kt.get(k2)).collect() == encode_fn('abc')
assert await kt.get_all(k2) == encode_fn('abc')
await kt.put(k1, encode_fn('abc'))
assert await ds.get_all(k1) == encode_fn('abc')
assert await ds.get_all(k2) == encode_fn('abc')
assert await kt.get_all(k1) == encode_fn('abc')
assert await kt.get_all(k2) == encode_fn('abc')
await ds.delete(k1)
assert not await ds.contains(k1)
assert await ds.get_all(k2) == encode_fn('abc')
assert await kt.get_all(k1) == encode_fn('abc')
assert not await kt.contains(k2)
await kt.delete(k1)
assert not await ds.contains(k1)
assert not await ds.contains(k2)
assert not await kt.contains(k1)
assert not await kt.contains(k2)
async def test_stats_concurrent(temp_path):
async with trio.open_nursery() as nursery:
queue = queue_.Queue()
async with FileSystemDatastore.create(temp_path, stats=True) as fs:
assert fs.datastore_stats().size == 0
assert fs.datastore_stats().size_accuracy == "exact"
await fs.put(datastore.Key("/a"), b"1234")
assert fs.datastore_stats().size == 4
await fs.flush()
assert fs.datastore_stats().size == 4
# Start concurrent datastore on separate thread
queue.put(("start", None, None))
nursery.start_soon(trio.to_thread.run_sync, trio.run,
concurrent_datastore, temp_path, queue)
try:
# Wait for thread to read dummy value from queue
while not queue.empty():
await trio.sleep(0)
# Perform concurrent write
queue.put(("put", datastore.Key("/b"), b"Hallo Welt"))
queue.put(("put_new", datastore.Key("/"), b"Hallo Welt"))
queue.join()
# Our datastore wouldn't know about it yet
assert fs.datastore_stats().size == 4
# Flush the remote datastore
queue.put(("flush", None, None))
queue.join()
# Our datastore still wouldn't know about it yet
assert fs.datastore_stats().size == 4
# Write something more into our datastore
await fs.put(datastore.Key("/c"), b"1234")
key_new = await fs.put_new(datastore.Key("/"), b"1234")
# Our datastore still will still only know about its changes
assert fs.datastore_stats().size == 12
# Let's replace "/c"
await fs.rename(key_new, datastore.Key("/c"))
# Our datastore still will still only know about its changes
assert fs.datastore_stats().size == 8
# Flush our datastore after theirs
await fs.flush()
# Now we see their change
assert fs.datastore_stats().size == 28
finally:
queue.put(("quit", None, None))
def key(self, item):
return datastore.Key(item)
def _untransform_key(self, key: datastore.Key) -> datastore.Key:
"""Returns `key` with the namespace stripped"""
return datastore.Key(key.list[len(self.namespace):])
