Ejemplo n.º 1
0
  def test_type(self):
    k1 = Key('/A/B/C:c')
    k2 = Key('/A/B/C:c/D:d')

    self.assertRaises(TypeError, k1.isAncestorOf, str(k2))
    self.assertTrue(k1.isAncestorOf(k2))
    self.assertTrue(k2.isDescendantOf(k1))
    self.assertEqual(k1.type, 'C')
    self.assertEqual(k2.type, 'D')
    self.assertEqual(k1.type, k2.parent.type)
Ejemplo n.º 2
0
  def __subtest_basic(self, string):
    fixedString = Key.removeDuplicateSlashes(string)
    lastNamespace = fixedString.rsplit('/')[-1].split(':')
    ktype = lastNamespace[0] if len(lastNamespace) > 1 else ''
    name = lastNamespace[-1]
    path = fixedString.rsplit('/', 1)[0] + '/' + ktype
    instance = fixedString + ':' + 'c'

    self.assertEqual(Key(string)._string, fixedString)
    self.assertEqual(Key(string), Key(string))
    self.assertEqual(str(Key(string)), fixedString)
    self.assertEqual(repr(Key(string)), "Key('%s')" % fixedString)
    self.assertEqual(Key(string).name, name)
    self.assertEqual(Key(string).type, ktype)
    self.assertEqual(Key(string).instance('c'), Key(instance))
    self.assertEqual(Key(string).path, Key(path))
    self.assertEqual(Key(string), eval(repr(Key(string))))

    self.assertTrue(Key(string).child('a') > Key(string))
    self.assertTrue(Key(string).child('a') < Key(string).child('b'))
    self.assertTrue(Key(string) == Key(string))

    self.assertRaises(TypeError, cmp, Key(string), string)

    split = fixedString.split('/')
    if len(split) > 1:
      self.assertEqual(Key('/'.join(split[:-1])), Key(string).parent)
    else:
      self.assertRaises(ValueError, lambda: Key(string).parent)

    namespace = split[-1].split(':')
    if len(namespace) > 1:
      self.assertEqual(namespace[0], Key(string).type)
    else:
      self.assertEqual('', Key(string).type)
Ejemplo n.º 3
0
 def test_random(self):
   keys = set()
   for i in range(0, 1000):
     random = Key.randomKey()
     self.assertFalse(random in keys)
     keys.add(random)
   self.assertEqual(len(keys), 1000)
Ejemplo n.º 4
0
async def test_keytransform_reverse_transform(Adapter, DictDatastore, encode_fn):
	def transform(key):
		return key.reverse

	ds = DictDatastore()
	kt = Adapter(ds, key_transform=transform)

	k1 = Key('/a/b/c')
	k2 = Key('/c/b/a')
	assert not await ds.contains(k1)
	assert not await ds.contains(k2)
	assert not await kt.contains(k1)
	assert not await kt.contains(k2)

	await ds.put(k1, encode_fn('abc'))
	assert await ds.get_all(k1) == encode_fn('abc')
	assert not await ds.contains(k2)
	assert not await kt.contains(k1)
	assert await kt.get_all(k2) == encode_fn('abc')

	await kt.put(k1, encode_fn('abc'))
	assert await ds.get_all(k1) == encode_fn('abc')
	assert await ds.get_all(k2) == encode_fn('abc')
	assert await kt.get_all(k1) == encode_fn('abc')
	assert await kt.get_all(k2) == encode_fn('abc')

	await ds.delete(k1)
	assert not await ds.contains(k1)
	assert await ds.get_all(k2) == encode_fn('abc')
	assert await kt.get_all(k1) == encode_fn('abc')
	assert not await kt.contains(k2)

	await kt.delete(k1)
	assert not await ds.contains(k1)
	assert not await ds.contains(k2)
	assert not await kt.contains(k1)
	assert not await kt.contains(k2)
Ejemplo n.º 5
0
    async def _put(self, key: datastore.Key,
                   value: datastore.abc.ReceiveChannel[T_co],
                   **kwargs: typing.Any) -> None:
        """Stores the object `value` named by `key`.
		   DirectoryTreeDatastore stores a directory entry.
		"""
        await super()._put(key, value, **kwargs)

        # ignore root
        if key.is_top_level():
            return

        # Add entry to directory
        dir_key = key.parent.instance('directory')
        await super().directory_add(dir_key, key, create=True)
Ejemplo n.º 6
0
 def withData(cls, data):
     '''Constructs a version of this model with given data'''
     key = data[cls.key_attr]
     instance = cls(Key(key))
     instance.updateData(data)
     return instance
Ejemplo n.º 7
0
async def test_sharded(DatastoreTests, Adapter, DictDatastore, encode_fn):
    numelems = 100

    s1 = DictDatastore()
    s2 = DictDatastore()
    s3 = DictDatastore()
    s4 = DictDatastore()
    s5 = DictDatastore()
    stores = [s1, s2, s3, s4, s5]

    def hash(key):
        return int(key.name) * len(stores) // numelems

    sharded = Adapter(stores, sharding_fn=hash)

    def sumlens(stores):
        return sum(map(lambda s: len(s), stores))

    async def checkFor(key, value, sharded, shard=None):
        correct_shard = sharded._stores[hash(key) % len(sharded._stores)]

        for s in sharded._stores:
            if shard and s == shard:
                assert await s.contains(key)
                assert await s.get_all(key) == encode_fn(value)
            else:
                assert not await s.contains(key)

        if correct_shard == shard:
            assert await sharded.contains(key)
            assert await sharded.get_all(key) == encode_fn(value)
        else:
            assert not await sharded.contains(key)

    assert sumlens(stores) == 0
    # test all correct.
    for value in range(0, numelems):
        key = Key(f"/fdasfdfdsafdsafdsa/{value}")
        shard = stores[hash(key) % len(stores)]
        await checkFor(key, value, sharded)
        await shard.put(key, encode_fn(value))
        await checkFor(key, value, sharded, shard)
    assert sumlens(stores) == numelems

    # ensure its in the same spots.
    for i in range(0, numelems):
        key = Key(f"/fdasfdfdsafdsafdsa/{value}")
        shard = stores[hash(key) % len(stores)]
        await checkFor(key, value, sharded, shard)
        await shard.put(key, encode_fn(value))
        await checkFor(key, value, sharded, shard)
    assert sumlens(stores) == numelems

    # ensure its in the same spots.
    for value in range(0, numelems):
        key = Key(f"/fdasfdfdsafdsafdsa/{value}")
        shard = stores[hash(key) % len(stores)]
        await checkFor(key, value, sharded, shard)
        await sharded.put(key, encode_fn(value))
        await checkFor(key, value, sharded, shard)
    assert sumlens(stores) == numelems

    # ensure its in the same spots.
    for value in range(0, numelems):
        key = Key(f"/fdasfdfdsafdsafdsa/{value}")
        shard = stores[hash(key) % len(stores)]
        await checkFor(key, value, sharded, shard)
        if value % 2 == 0:
            await shard.delete(key)
        else:
            await sharded.delete(key)
        await checkFor(key, value, sharded)
    assert sumlens(stores) == 0

    # try out adding it to the wrong shards.
    for value in range(0, numelems):
        key = Key(f"/fdasfdfdsafdsafdsa/{value}")
        incorrect_shard = stores[(hash(key) + 1) % len(stores)]
        await checkFor(key, value, sharded)
        await incorrect_shard.put(key, encode_fn(value))
        await checkFor(key, value, sharded, incorrect_shard)
    assert sumlens(stores) == numelems

    # ensure its in the same spots.
    for value in range(0, numelems):
        key = Key(f"/fdasfdfdsafdsafdsa/{value}")
        incorrect_shard = stores[(hash(key) + 1) % len(stores)]
        await checkFor(key, value, sharded, incorrect_shard)
        await incorrect_shard.put(key, encode_fn(value))
        await checkFor(key, value, sharded, incorrect_shard)
    assert sumlens(stores) == numelems

    # this wont do anything
    for value in range(0, numelems):
        key = Key(f"/fdasfdfdsafdsafdsa/{value}")
        incorrect_shard = stores[(hash(key) + 1) % len(stores)]
        await checkFor(key, value, sharded, incorrect_shard)
        with pytest.raises(KeyError):
            await sharded.delete(key)
        await checkFor(key, value, sharded, incorrect_shard)
    assert sumlens(stores) == numelems

    # this will place it correctly.
    for value in range(0, numelems):
        key = Key(f"/fdasfdfdsafdsafdsa/{value}")
        incorrect_shard = stores[(hash(key) + 1) % len(stores)]
        correct_shard = stores[(hash(key)) % len(stores)]
        await checkFor(key, value, sharded, incorrect_shard)
        await sharded.put(key, encode_fn(value))
        await incorrect_shard.delete(key)
        await checkFor(key, value, sharded, correct_shard)
    assert sumlens(stores) == numelems

    # this will place it correctly.
    for value in range(0, numelems):
        key = Key(f"/fdasfdfdsafdsafdsa/{value}")
        correct_shard = stores[(hash(key)) % len(stores)]
        await checkFor(key, value, sharded, correct_shard)
        await sharded.delete(key)
        await checkFor(key, value, sharded)
    assert sumlens(stores) == 0

    await DatastoreTests([sharded]).subtest_simple()
Ejemplo n.º 8
0
async def test_tiered(DatastoreTests, Adapter, DictDatastore, encode_fn):
    s1 = DictDatastore()
    s2 = DictDatastore()
    s3 = DictDatastore()
    ts = Adapter([s1, s2, s3])

    k1 = Key('1')
    k2 = Key('2')
    k3 = Key('3')

    await s1.put(k1, encode_fn('1'))
    await s2.put(k2, encode_fn('2'))
    await s3.put(k3, encode_fn('3'))

    assert await s1.contains(k1)
    assert not await s2.contains(k1)
    assert not await s3.contains(k1)
    assert await ts.contains(k1)

    assert await ts.get_all(k1) == encode_fn('1')
    assert await s1.get_all(k1) == encode_fn('1')
    assert not await s2.contains(k1)
    assert not await s3.contains(k1)

    assert not await s1.contains(k2)
    assert await s2.contains(k2)
    assert not await s3.contains(k2)
    assert await ts.contains(k2)

    assert await s2.get_all(k2) == encode_fn('2')
    assert not await s1.contains(k2)
    assert not await s3.contains(k2)

    # Read value from TS (where it will be found in T2) and check whether it was
    # copied into T1 because of this
    assert await ts.get_all(k2) == encode_fn('2')
    assert await s1.get_all(k2) == encode_fn('2')
    assert await s2.get_all(k2) == encode_fn('2')
    assert not await s3.contains(k2)

    assert not await s1.contains(k3)
    assert not await s2.contains(k3)
    assert await s3.contains(k3)
    assert await ts.contains(k3)

    assert await s3.get_all(k3) == encode_fn('3')
    assert not await s1.contains(k3)
    assert not await s2.contains(k3)

    assert await ts.get_all(k3) == encode_fn('3')
    assert await s1.get_all(k3) == encode_fn('3')
    assert await s2.get_all(k3) == encode_fn('3')
    assert await s3.get_all(k3) == encode_fn('3')

    await ts.delete(k1)
    await ts.delete(k2)
    await ts.delete(k3)

    assert not await ts.contains(k1)
    assert not await ts.contains(k2)
    assert not await ts.contains(k3)

    await DatastoreTests([ts]).subtest_simple()
Ejemplo n.º 9
0
 def transform(key):
     return Key(str(key).lower())
Ejemplo n.º 10
0
    async def _put_new_indirect(
        self,
        prefix: datastore.Key  # type: ignore[override]
    ) -> datastore.abc.BinaryDatastore._PUT_NEW_INDIRECT_RT:
        """Stores data in a new subkey below *prefix*
		
		Arguments
		---------
		prefix
			Key below which to create a new binary data slot to store data at
		
		Raises
		------
		RuntimeError
			The given *prefix* names a value, not a subtree OR contains a
			value item as part of the key path
		"""
        # Validate that the key is well-formed
        #
        # Usage of assert here will cause this call to be optimized away in `-O` mode.
        assert self.verify_key_valid(prefix)

        path_dir = trio.Path(self.object_path(prefix, suffix=False))
        path_prefix_t = ".tmp-new-"
        path_prefix_f = ".new-"

        # Ensure containing directory exists
        try:
            await path_dir.mkdir(parents=True, exist_ok=True)
        except FileExistsError as exc:
            # Should hopefully only happen if `object_extension` is `""`
            raise RuntimeError(
                f"Adding a new file below \"{path_dir}\" requires that path "
                f"to not be a value") from exc

        # Create target file
        target_file = await make_named_tempfile(mode="wb",
                                                dir=path_dir,
                                                prefix=path_prefix_f,
                                                suffix=self.object_extension,
                                                delete=False)

        # Write to extra temporary file if stats tracking to account for collisions, else write
        # directly to target file
        if self._stats is not None:
            try:
                file = await make_named_tempfile(mode="wb",
                                                 dir=path_dir,
                                                 prefix=path_prefix_t,
                                                 delete=False)
            finally:
                # This file will not be directly written to
                await target_file.aclose()
        else:
            file = target_file

        # Return a pretty uninspired callback function that reads data from the stream
        # and writes it to the target file
        async def callback(value: datastore.abc.ReceiveStream) -> None:
            try:
                await self._receive_and_write(file, value)
            except BaseException:
                try:
                    await file.aclose()
                finally:
                    with trio.CancelScope(shield=True):
                        await file.unlink()
                raise
            else:
                await file.aclose()

                if self._stats is not None:
                    async with self._stats_lock:  # type: ignore[union-attr]
                        await self._put_replace(file.name, target_file.name)

        return prefix.child(
            pathlib.Path(
                target_file.name).name[:-len(self.object_extension)]), callback
Ejemplo n.º 11
0
  def test_ancestry(self):
    k1 = Key('/A/B/C')
    k2 = Key('/A/B/C/D')

    self.assertEqual(k1._string, '/A/B/C')
    self.assertEqual(k2._string, '/A/B/C/D')
    self.assertTrue(k1.isAncestorOf(k2))
    self.assertTrue(k2.isDescendantOf(k1))
    self.assertTrue(Key('/A').isAncestorOf(k2))
    self.assertTrue(Key('/A').isAncestorOf(k1))
    self.assertFalse(Key('/A').isDescendantOf(k2))
    self.assertFalse(Key('/A').isDescendantOf(k1))
    self.assertTrue(k2.isDescendantOf(Key('/A')))
    self.assertTrue(k1.isDescendantOf(Key('/A')))
    self.assertFalse(k2.isAncestorOf(Key('/A')))
    self.assertFalse(k1.isAncestorOf(Key('/A')))
    self.assertFalse(k2.isAncestorOf(k2))
    self.assertFalse(k1.isAncestorOf(k1))
    self.assertEqual(k1.child('D'), k2)
    self.assertEqual(k1, k2.parent)
    self.assertEqual(k1.path, k2.parent.path)