async def test_namespace(Adapter, DictDatastore, encode_fn): k1 = datastore.Key('/c/d') k2 = datastore.Key('/a/b') k3 = datastore.Key('/a/b/c/d') ds = DictDatastore() async with Adapter(k2, ds) as nd: await ds.put(k1, encode_fn('cd')) await ds.put(k3, encode_fn('abcd')) assert await ds.get_all(k1) == encode_fn('cd') assert not await ds.contains(k2) assert await ds.get_all(k3) == encode_fn('abcd') assert await nd.get_all(k1) == encode_fn('abcd') assert not await nd.contains(k2) assert not await nd.contains(k3) async def test(key, val): await nd.put(key, val) assert await nd.get_all(key) == val assert not await ds.contains(key) assert not await nd.contains(k2.child(key)) assert await ds.get_all(k2.child(key)) == val for i in range(0, 10): await test(datastore.Key(str(i)), encode_fn(f"val{i}"))
async def test_lowercase_key(Adapter, DictDatastore, encode_fn): ds = DictDatastore() async with Adapter(ds) as lds: k1 = datastore.Key('hello') k2 = datastore.Key('HELLO') k3 = datastore.Key('HeLlo') await ds.put(k1, encode_fn('world')) await ds.put(k2, encode_fn('WORLD')) assert await ds.get_all(k1) == encode_fn('world') assert await ds.get_all(k2) == encode_fn('WORLD') assert not await ds.contains(k3) assert await lds.get_all(k1) == encode_fn('world') assert await lds.get_all(k2) == encode_fn('world') assert await lds.get_all(k3) == encode_fn('world') async def test(key, val): await lds.put(key, val) assert await lds.get_all(k1) == val assert await lds.get_all(k2) == val assert await lds.get_all(k3) == val await test(k1, encode_fn('a')) await test(k2, encode_fn('b')) await test(k3, encode_fn('c'))
async def test_stats_tracking(temp_path): async with FileSystemDatastore.create(temp_path, stats=True) as fs: assert fs.datastore_stats().size == 0 assert fs.datastore_stats().size_accuracy == "exact" # Write to datastore key await fs.put(datastore.Key("/a"), b"1234") assert fs.datastore_stats().size == 4 # Overwrite datastore key await fs.put(datastore.Key("/a"), b"Hallo Welt") assert fs.datastore_stats().size == 10 # Create new datastore key key_new = await fs.put_new(datastore.Key("/"), b"TC") assert fs.datastore_stats().size == 12 # Remove datastore keys await fs.delete(datastore.Key("/a")) assert fs.datastore_stats().size == 2 await fs.delete(key_new) assert fs.datastore_stats().size == 0
async def test_nested_path_3_2(Adapter, DictDatastore, encode_fn): opts = {} opts['k1'] = datastore.Key('/abcdefghijk') opts['k2'] = datastore.Key('/abcdefghijki') opts['k3'] = datastore.Key('/ab/cd/ef/abcdefghijk') opts['k4'] = datastore.Key('/ab/cd/ef/abcdefghijki') opts['depth'] = 3 opts['length'] = 2 await subtest_nested_path_ds(Adapter, DictDatastore, encode_fn, **opts)
async def test_namespace_simple(DatastoreTests, Adapter, DictDatastore, encode_fn): async with contextlib.AsyncExitStack() as stack: s1 = stack.push_async_exit(Adapter(datastore.Key('a'), DictDatastore())) s2 = stack.push_async_exit(Adapter(datastore.Key('b'), DictDatastore())) s3 = stack.push_async_exit(Adapter(datastore.Key('c'), DictDatastore())) stores = [s1, s2, s3] await DatastoreTests(stores, test_put_new=True).subtest_simple()
async def test_stats_restore(temp_path): async with FileSystemDatastore.create(temp_path, stats=True) as fs: assert fs.datastore_stats().size == 0 assert fs.datastore_stats().size_accuracy == "exact" await fs.put(datastore.Key("/a"), b"1234") assert fs.datastore_stats().size == 4 # Re-open datastore and check that the stats are still there async with FileSystemDatastore.create(temp_path, stats=True) as fs: assert fs.datastore_stats().size == 4 assert fs.datastore_stats().size_accuracy == "exact" # Replace content with stuff written by a non-cooperating datastore user await (trio.Path(temp_path) / "diskUsage.data").write_text( json.dumps({ "diskUsage": 3, "accuracy": "initial-exact" }), encoding="utf-8") # Re-open datastore and check that the non-cooperating stats data is # properly merged async with FileSystemDatastore.create(temp_path, stats=True) as fs: assert fs.datastore_stats().size == 7 assert fs.datastore_stats().size_accuracy == "exact"
def _find_mountpoint(self, key: datastore.Key) \ -> typing.Tuple[typing.Optional[DS], datastore.Key, datastore.Key]: current = self.mounts ds = None offset = 0 for idx, part in enumerate(map(str, key.list)): mount = current.get(part) if mount is None: break if mount[1] is not None: ds = mount[1] offset = idx current = mount[0] return ds, datastore.Key(key.list[:offset]), datastore.Key( key.list[(offset + 1):])
async def create( cls, root: typing.Union[os_PathLike_str, str], *, case_sensitive: bool = True, remove_empty: bool = True, stats: bool = False, stats_key: datastore.Key = DEFAULT_STATS_KEY ) -> 'FileSystemDatastore': """Initialize the datastore with given root directory `root`. Arguments --------- root A path at which to mount this filesystem datastore. case_sensitive Keep case of all path items (True) or lower case them (False) before passing them to the OS. Note that, if the underlying file system is case-insensitive this option will not prevent conflicts between paths that differ in case only. remove_empty Attempt to remove empty directories in the underlying file system. While this is enabled every successful delete operation will be followed by at least one extra context switch to invoke the `rmdir` system call. stats Track summary statistics about the contents of the datastore, currently only the total apparent size of all files on the disk is tracked stats_key The key/filepath at which to persist statistics between runs """ if not root: raise ValueError( 'root path must not be empty (use \'.\' for current directory)' ) # Ensure target directory exists await trio.Path(root).mkdir(parents=True, exist_ok=True) # Create instance self = cls(_create_call=True) # Do the usual constructor stuff self.root_path = pathlib.PurePath(root) self.case_sensitive = bool(case_sensitive) self.remove_empty = bool(remove_empty) self.stats_key = datastore.Key(stats_key) # Enable stats processing if stats: self._stats_lock = trio.Lock() await self._init_stats() return self
async def test_keytransform_reverse_transform(Adapter, DictDatastore, encode_fn): def transform(key): return key.reverse ds = DictDatastore() async with Adapter(ds, key_transform=transform) as kt: k1 = datastore.Key('/a/b/c') k2 = datastore.Key('/c/b/a') assert not await ds.contains(k1) assert not await ds.contains(k2) assert not await kt.contains(k1) assert not await kt.contains(k2) await ds.put(k1, encode_fn('abc')) assert await ds.get_all(k1) == encode_fn('abc') assert not await ds.contains(k2) assert not await kt.contains(k1) with pytest.raises(KeyError): await (await kt.get(k1)).aclose() assert await (await kt.get(k2)).collect() == encode_fn('abc') assert await kt.get_all(k2) == encode_fn('abc') await kt.put(k1, encode_fn('abc')) assert await ds.get_all(k1) == encode_fn('abc') assert await ds.get_all(k2) == encode_fn('abc') assert await kt.get_all(k1) == encode_fn('abc') assert await kt.get_all(k2) == encode_fn('abc') await ds.delete(k1) assert not await ds.contains(k1) assert await ds.get_all(k2) == encode_fn('abc') assert await kt.get_all(k1) == encode_fn('abc') assert not await kt.contains(k2) await kt.delete(k1) assert not await ds.contains(k1) assert not await ds.contains(k2) assert not await kt.contains(k1) assert not await kt.contains(k2)
async def test_stats_concurrent(temp_path): async with trio.open_nursery() as nursery: queue = queue_.Queue() async with FileSystemDatastore.create(temp_path, stats=True) as fs: assert fs.datastore_stats().size == 0 assert fs.datastore_stats().size_accuracy == "exact" await fs.put(datastore.Key("/a"), b"1234") assert fs.datastore_stats().size == 4 await fs.flush() assert fs.datastore_stats().size == 4 # Start concurrent datastore on separate thread queue.put(("start", None, None)) nursery.start_soon(trio.to_thread.run_sync, trio.run, concurrent_datastore, temp_path, queue) try: # Wait for thread to read dummy value from queue while not queue.empty(): await trio.sleep(0) # Perform concurrent write queue.put(("put", datastore.Key("/b"), b"Hallo Welt")) queue.put(("put_new", datastore.Key("/"), b"Hallo Welt")) queue.join() # Our datastore wouldn't know about it yet assert fs.datastore_stats().size == 4 # Flush the remote datastore queue.put(("flush", None, None)) queue.join() # Our datastore still wouldn't know about it yet assert fs.datastore_stats().size == 4 # Write something more into our datastore await fs.put(datastore.Key("/c"), b"1234") key_new = await fs.put_new(datastore.Key("/"), b"1234") # Our datastore still will still only know about its changes assert fs.datastore_stats().size == 12 # Let's replace "/c" await fs.rename(key_new, datastore.Key("/c")) # Our datastore still will still only know about its changes assert fs.datastore_stats().size == 8 # Flush our datastore after theirs await fs.flush() # Now we see their change assert fs.datastore_stats().size == 28 finally: queue.put(("quit", None, None))
def key(self, item): return datastore.Key(item)
def _untransform_key(self, key: datastore.Key) -> datastore.Key: """Returns `key` with the namespace stripped""" return datastore.Key(key.list[len(self.namespace):])