def test_type(self):
k1 = Key('/A/B/C:c')
k2 = Key('/A/B/C:c/D:d')
self.assertRaises(TypeError, k1.isAncestorOf, str(k2))
self.assertTrue(k1.isAncestorOf(k2))
self.assertTrue(k2.isDescendantOf(k1))
self.assertEqual(k1.type, 'C')
self.assertEqual(k2.type, 'D')
self.assertEqual(k1.type, k2.parent.type)
def __subtest_basic(self, string):
fixedString = Key.removeDuplicateSlashes(string)
lastNamespace = fixedString.rsplit('/')[-1].split(':')
ktype = lastNamespace[0] if len(lastNamespace) > 1 else ''
name = lastNamespace[-1]
path = fixedString.rsplit('/', 1)[0] + '/' + ktype
instance = fixedString + ':' + 'c'
self.assertEqual(Key(string)._string, fixedString)
self.assertEqual(Key(string), Key(string))
self.assertEqual(str(Key(string)), fixedString)
self.assertEqual(repr(Key(string)), "Key('%s')" % fixedString)
self.assertEqual(Key(string).name, name)
self.assertEqual(Key(string).type, ktype)
self.assertEqual(Key(string).instance('c'), Key(instance))
self.assertEqual(Key(string).path, Key(path))
self.assertEqual(Key(string), eval(repr(Key(string))))
self.assertTrue(Key(string).child('a') > Key(string))
self.assertTrue(Key(string).child('a') < Key(string).child('b'))
self.assertTrue(Key(string) == Key(string))
self.assertRaises(TypeError, cmp, Key(string), string)
split = fixedString.split('/')
if len(split) > 1:
self.assertEqual(Key('/'.join(split[:-1])), Key(string).parent)
else:
self.assertRaises(ValueError, lambda: Key(string).parent)
namespace = split[-1].split(':')
if len(namespace) > 1:
self.assertEqual(namespace[0], Key(string).type)
else:
self.assertEqual('', Key(string).type)
def test_random(self):
keys = set()
for i in range(0, 1000):
random = Key.randomKey()
self.assertFalse(random in keys)
keys.add(random)
self.assertEqual(len(keys), 1000)
async def test_keytransform_reverse_transform(Adapter, DictDatastore, encode_fn):
def transform(key):
return key.reverse
ds = DictDatastore()
kt = Adapter(ds, key_transform=transform)
k1 = Key('/a/b/c')
k2 = Key('/c/b/a')
assert not await ds.contains(k1)
assert not await ds.contains(k2)
assert not await kt.contains(k1)
assert not await kt.contains(k2)
await ds.put(k1, encode_fn('abc'))
assert await ds.get_all(k1) == encode_fn('abc')
assert not await ds.contains(k2)
assert not await kt.contains(k1)
assert await kt.get_all(k2) == encode_fn('abc')
await kt.put(k1, encode_fn('abc'))
assert await ds.get_all(k1) == encode_fn('abc')
assert await ds.get_all(k2) == encode_fn('abc')
assert await kt.get_all(k1) == encode_fn('abc')
assert await kt.get_all(k2) == encode_fn('abc')
await ds.delete(k1)
assert not await ds.contains(k1)
assert await ds.get_all(k2) == encode_fn('abc')
assert await kt.get_all(k1) == encode_fn('abc')
assert not await kt.contains(k2)
await kt.delete(k1)
assert not await ds.contains(k1)
assert not await ds.contains(k2)
assert not await kt.contains(k1)
assert not await kt.contains(k2)
async def _put(self, key: datastore.Key,
value: datastore.abc.ReceiveChannel[T_co],
**kwargs: typing.Any) -> None:
"""Stores the object `value` named by `key`.
DirectoryTreeDatastore stores a directory entry.
"""
await super()._put(key, value, **kwargs)
# ignore root
if key.is_top_level():
return
# Add entry to directory
dir_key = key.parent.instance('directory')
await super().directory_add(dir_key, key, create=True)
def withData(cls, data):
'''Constructs a version of this model with given data'''
key = data[cls.key_attr]
instance = cls(Key(key))
instance.updateData(data)
return instance
async def test_sharded(DatastoreTests, Adapter, DictDatastore, encode_fn):
numelems = 100
s1 = DictDatastore()
s2 = DictDatastore()
s3 = DictDatastore()
s4 = DictDatastore()
s5 = DictDatastore()
stores = [s1, s2, s3, s4, s5]
def hash(key):
return int(key.name) * len(stores) // numelems
sharded = Adapter(stores, sharding_fn=hash)
def sumlens(stores):
return sum(map(lambda s: len(s), stores))
async def checkFor(key, value, sharded, shard=None):
correct_shard = sharded._stores[hash(key) % len(sharded._stores)]
for s in sharded._stores:
if shard and s == shard:
assert await s.contains(key)
assert await s.get_all(key) == encode_fn(value)
else:
assert not await s.contains(key)
if correct_shard == shard:
assert await sharded.contains(key)
assert await sharded.get_all(key) == encode_fn(value)
else:
assert not await sharded.contains(key)
assert sumlens(stores) == 0
# test all correct.
for value in range(0, numelems):
key = Key(f"/fdasfdfdsafdsafdsa/{value}")
shard = stores[hash(key) % len(stores)]
await checkFor(key, value, sharded)
await shard.put(key, encode_fn(value))
await checkFor(key, value, sharded, shard)
assert sumlens(stores) == numelems
# ensure its in the same spots.
for i in range(0, numelems):
key = Key(f"/fdasfdfdsafdsafdsa/{value}")
shard = stores[hash(key) % len(stores)]
await checkFor(key, value, sharded, shard)
await shard.put(key, encode_fn(value))
await checkFor(key, value, sharded, shard)
assert sumlens(stores) == numelems
# ensure its in the same spots.
for value in range(0, numelems):
key = Key(f"/fdasfdfdsafdsafdsa/{value}")
shard = stores[hash(key) % len(stores)]
await checkFor(key, value, sharded, shard)
await sharded.put(key, encode_fn(value))
await checkFor(key, value, sharded, shard)
assert sumlens(stores) == numelems
# ensure its in the same spots.
for value in range(0, numelems):
key = Key(f"/fdasfdfdsafdsafdsa/{value}")
shard = stores[hash(key) % len(stores)]
await checkFor(key, value, sharded, shard)
if value % 2 == 0:
await shard.delete(key)
else:
await sharded.delete(key)
await checkFor(key, value, sharded)
assert sumlens(stores) == 0
# try out adding it to the wrong shards.
for value in range(0, numelems):
key = Key(f"/fdasfdfdsafdsafdsa/{value}")
incorrect_shard = stores[(hash(key) + 1) % len(stores)]
await checkFor(key, value, sharded)
await incorrect_shard.put(key, encode_fn(value))
await checkFor(key, value, sharded, incorrect_shard)
assert sumlens(stores) == numelems
# ensure its in the same spots.
for value in range(0, numelems):
key = Key(f"/fdasfdfdsafdsafdsa/{value}")
incorrect_shard = stores[(hash(key) + 1) % len(stores)]
await checkFor(key, value, sharded, incorrect_shard)
await incorrect_shard.put(key, encode_fn(value))
await checkFor(key, value, sharded, incorrect_shard)
assert sumlens(stores) == numelems
# this wont do anything
for value in range(0, numelems):
key = Key(f"/fdasfdfdsafdsafdsa/{value}")
incorrect_shard = stores[(hash(key) + 1) % len(stores)]
await checkFor(key, value, sharded, incorrect_shard)
with pytest.raises(KeyError):
await sharded.delete(key)
await checkFor(key, value, sharded, incorrect_shard)
assert sumlens(stores) == numelems
# this will place it correctly.
for value in range(0, numelems):
key = Key(f"/fdasfdfdsafdsafdsa/{value}")
incorrect_shard = stores[(hash(key) + 1) % len(stores)]
correct_shard = stores[(hash(key)) % len(stores)]
await checkFor(key, value, sharded, incorrect_shard)
await sharded.put(key, encode_fn(value))
await incorrect_shard.delete(key)
await checkFor(key, value, sharded, correct_shard)
assert sumlens(stores) == numelems
# this will place it correctly.
for value in range(0, numelems):
key = Key(f"/fdasfdfdsafdsafdsa/{value}")
correct_shard = stores[(hash(key)) % len(stores)]
await checkFor(key, value, sharded, correct_shard)
await sharded.delete(key)
await checkFor(key, value, sharded)
assert sumlens(stores) == 0
await DatastoreTests([sharded]).subtest_simple()
async def test_tiered(DatastoreTests, Adapter, DictDatastore, encode_fn):
s1 = DictDatastore()
s2 = DictDatastore()
s3 = DictDatastore()
ts = Adapter([s1, s2, s3])
k1 = Key('1')
k2 = Key('2')
k3 = Key('3')
await s1.put(k1, encode_fn('1'))
await s2.put(k2, encode_fn('2'))
await s3.put(k3, encode_fn('3'))
assert await s1.contains(k1)
assert not await s2.contains(k1)
assert not await s3.contains(k1)
assert await ts.contains(k1)
assert await ts.get_all(k1) == encode_fn('1')
assert await s1.get_all(k1) == encode_fn('1')
assert not await s2.contains(k1)
assert not await s3.contains(k1)
assert not await s1.contains(k2)
assert await s2.contains(k2)
assert not await s3.contains(k2)
assert await ts.contains(k2)
assert await s2.get_all(k2) == encode_fn('2')
assert not await s1.contains(k2)
assert not await s3.contains(k2)
# Read value from TS (where it will be found in T2) and check whether it was
# copied into T1 because of this
assert await ts.get_all(k2) == encode_fn('2')
assert await s1.get_all(k2) == encode_fn('2')
assert await s2.get_all(k2) == encode_fn('2')
assert not await s3.contains(k2)
assert not await s1.contains(k3)
assert not await s2.contains(k3)
assert await s3.contains(k3)
assert await ts.contains(k3)
assert await s3.get_all(k3) == encode_fn('3')
assert not await s1.contains(k3)
assert not await s2.contains(k3)
assert await ts.get_all(k3) == encode_fn('3')
assert await s1.get_all(k3) == encode_fn('3')
assert await s2.get_all(k3) == encode_fn('3')
assert await s3.get_all(k3) == encode_fn('3')
await ts.delete(k1)
await ts.delete(k2)
await ts.delete(k3)
assert not await ts.contains(k1)
assert not await ts.contains(k2)
assert not await ts.contains(k3)
await DatastoreTests([ts]).subtest_simple()
def transform(key):
return Key(str(key).lower())
async def _put_new_indirect(
self,
prefix: datastore.Key # type: ignore[override]
) -> datastore.abc.BinaryDatastore._PUT_NEW_INDIRECT_RT:
"""Stores data in a new subkey below *prefix*
Arguments
---------
prefix
Key below which to create a new binary data slot to store data at
Raises
------
RuntimeError
The given *prefix* names a value, not a subtree OR contains a
value item as part of the key path
"""
# Validate that the key is well-formed
#
# Usage of assert here will cause this call to be optimized away in `-O` mode.
assert self.verify_key_valid(prefix)
path_dir = trio.Path(self.object_path(prefix, suffix=False))
path_prefix_t = ".tmp-new-"
path_prefix_f = ".new-"
# Ensure containing directory exists
try:
await path_dir.mkdir(parents=True, exist_ok=True)
except FileExistsError as exc:
# Should hopefully only happen if `object_extension` is `""`
raise RuntimeError(
f"Adding a new file below \"{path_dir}\" requires that path "
f"to not be a value") from exc
# Create target file
target_file = await make_named_tempfile(mode="wb",
dir=path_dir,
prefix=path_prefix_f,
suffix=self.object_extension,
delete=False)
# Write to extra temporary file if stats tracking to account for collisions, else write
# directly to target file
if self._stats is not None:
try:
file = await make_named_tempfile(mode="wb",
dir=path_dir,
prefix=path_prefix_t,
delete=False)
finally:
# This file will not be directly written to
await target_file.aclose()
else:
file = target_file
# Return a pretty uninspired callback function that reads data from the stream
# and writes it to the target file
async def callback(value: datastore.abc.ReceiveStream) -> None:
try:
await self._receive_and_write(file, value)
except BaseException:
try:
await file.aclose()
finally:
with trio.CancelScope(shield=True):
await file.unlink()
raise
else:
await file.aclose()
if self._stats is not None:
async with self._stats_lock: # type: ignore[union-attr]
await self._put_replace(file.name, target_file.name)
return prefix.child(
pathlib.Path(
target_file.name).name[:-len(self.object_extension)]), callback
def test_ancestry(self):
k1 = Key('/A/B/C')
k2 = Key('/A/B/C/D')
self.assertEqual(k1._string, '/A/B/C')
self.assertEqual(k2._string, '/A/B/C/D')
self.assertTrue(k1.isAncestorOf(k2))
self.assertTrue(k2.isDescendantOf(k1))
self.assertTrue(Key('/A').isAncestorOf(k2))
self.assertTrue(Key('/A').isAncestorOf(k1))
self.assertFalse(Key('/A').isDescendantOf(k2))
self.assertFalse(Key('/A').isDescendantOf(k1))
self.assertTrue(k2.isDescendantOf(Key('/A')))
self.assertTrue(k1.isDescendantOf(Key('/A')))
self.assertFalse(k2.isAncestorOf(Key('/A')))
self.assertFalse(k1.isAncestorOf(Key('/A')))
self.assertFalse(k2.isAncestorOf(k2))
self.assertFalse(k1.isAncestorOf(k1))
self.assertEqual(k1.child('D'), k2)
self.assertEqual(k1, k2.parent)
self.assertEqual(k1.path, k2.parent.path)
