class NamespaceDatastore(KeyTransformDatastore):
"""Represents a simple ShimDatastore that namespaces all incoming keys.
For example:
>>> import datastore.core
>>>
>>> ds = datastore.DictDatastore()
>>> ds.put(datastore.Key('/a/b'), 'ab')
>>> ds.put(datastore.Key('/c/d'), 'cd')
>>> ds.put(datastore.Key('/a/b/c/d'), 'abcd')
>>>
>>> nd = datastore.NamespaceDatastore('/a/b', ds)
>>> nd.get(datastore.Key('/a/b'))
None
>>> nd.get(datastore.Key('/c/d'))
'abcd'
>>> nd.get(datastore.Key('/a/b/c/d'))
None
>>> nd.put(datastore.Key('/c/d'), 'cd')
>>> ds.get(datastore.Key('/a/b/c/d'))
'cd'
"""
def __init__(self, namespace, *args, **kwargs):
"""Initializes NamespaceDatastore with `key` namespace."""
super(NamespaceDatastore, self).__init__(*args, **kwargs)
self.keytransform = self.namespaceKey
self.namespace = Key(namespace)
def namespaceKey(self, key):
"""Returns a namespaced `key`: namespace.child(key)."""
return self.namespace.child(key)
def nestKey(self, key):
"""Returns a nested `key`."""
nest = self.nest_keyfn(key)
# if depth * length > len(key.name), we need to pad.
mult = 1 + int(self.nest_depth * self.nest_length / len(nest))
nest = nest * mult
pref = Key(self.nestedPath(nest, self.nest_depth, self.nest_length))
return pref.child(key)
def test_ancestry(self):
k1 = Key('/A/B/C')
k2 = Key('/A/B/C/D')
assert k1._string == '/A/B/C'
assert k2._string == '/A/B/C/D'
assert k1.is_ancestor_of(k2)
assert k2.is_descendant_of(k1)
assert Key('/A').is_ancestor_of(k2)
assert Key('/A').is_ancestor_of(k1)
assert not Key('/A').is_descendant_of(k2)
assert not Key('/A').is_descendant_of(k1)
assert k2.is_descendant_of(Key('/A'))
assert k1.is_descendant_of(Key('/A'))
assert not k2.is_ancestor_of(Key('/A'))
assert not k1.is_ancestor_of(Key('/A'))
assert not k2.is_ancestor_of(k2)
assert not k1.is_ancestor_of(k1)
assert k1.child('D') == k2
assert k1 == k2.parent
assert k1.path == k2.parent.path
def test_ancestry(self):
k1 = Key('/A/B/C')
k2 = Key('/A/B/C/D')
self.assertEqual(k1._string, '/A/B/C')
self.assertEqual(k2._string, '/A/B/C/D')
self.assertTrue(k1.is_ancestor_of(k2))
self.assertTrue(k2.is_descendant_of(k1))
self.assertTrue(Key('/A').is_ancestor_of(k2))
self.assertTrue(Key('/A').is_ancestor_of(k1))
self.assertFalse(Key('/A').is_descendant_of(k2))
self.assertFalse(Key('/A').is_descendant_of(k1))
self.assertTrue(k2.is_descendant_of(Key('/A')))
self.assertTrue(k1.is_descendant_of(Key('/A')))
self.assertFalse(k2.is_ancestor_of(Key('/A')))
self.assertFalse(k1.is_ancestor_of(Key('/A')))
self.assertFalse(k2.is_ancestor_of(k2))
self.assertFalse(k1.is_ancestor_of(k1))
self.assertEqual(k1.child('D'), k2)
self.assertEqual(k1, k2.parent)
self.assertEqual(k1.path, k2.parent.path)
def test_indexed_table(self):
# Assume that a table with indexes exists before we begin.
# The driver should see the indexes and use them
self.ds = DynamoDatastore(self.conn)
pkey = Key('/' + self.INDEXED_TABLE)
johnny_key = pkey.child('sales.Johnny')
johnny = {
'key': str(johnny_key),
'department': 'sales',
'name': 'Johnny',
'group': 'managers',
'age': 20,
'score': 1500
}
tom_key = pkey.child('sales.Tom')
tom = {
'key': str(tom_key),
'department': 'sales',
'name': 'Tom',
'group': 'employees',
'age': 30,
'score': 1000
}
barbara_key = pkey.child('online_marketing.Barbara')
barbara = {
'key': str(barbara_key),
'department': 'online_marketing',
'name': 'Barbara',
'group': 'managers',
'age': 40,
'score': 500
}
self.ds.put(johnny_key, johnny)
self.ds.put(tom_key, tom)
self.ds.put(barbara_key, barbara)
query = Query(pkey).filter('age','>',20)
# since the query does not contain any of the hash keys (department, username),
# we'll have to do a scan instead of a query
with mock.patch.object(Table, 'query', return_value=None) as mock_method:
res = list(self.ds.query(query))
assert not mock_method.called
assert res == [tom, barbara] or res == [barbara, tom]
# From here on out we don't want to use scan anymore
with mock.patch.object(Table, 'scan', return_value=None) as mock_method:
assert self.ds.get(johnny_key) == johnny
assert self.ds.get(tom_key) == tom
assert self.ds.get(barbara_key) == barbara
# Filter on hash key exclusively
q = Query(pkey).filter('department', '=', 'sales')
table = self.ds._table(q.key.child('_'))
idx = DynamoQuery.index_for_query(table, q)
assert idx.name is None, idx.name
assert idx.hash_key == 'department', idx.hash_key
assert idx.range_key == 'name', idx.range_key
res = list(self.ds.query(q))
assert res == [tom, johnny] or res == [johnny, tom]
# Filter on hash and range key
q = Query(pkey).filter('department', '=', 'sales').filter('name','=','Johnny')
res = list(self.ds.query(q))
assert res == [johnny]
# Filter on hash and range key with comparison
q = Query(pkey).filter('department', '=', 'sales').filter('name','>','Jason')
res = list(self.ds.query(q))
assert res == [johnny, tom] or res == [tom, johnny]
# Filter on a hash key and an arbitrary other non-indexed key
q = Query(pkey).filter('department', '=', 'sales').filter('age','>',25)
res = list(self.ds.query(q))
assert res == [tom]
# Filter on a hash key, secondary index, and non-indexed
q = Query(pkey).filter('department', '=', 'sales').filter('score','>=',25).filter('age','>=',25)
idx = DynamoQuery.index_for_query(table, q)
assert idx.name == 'ScoreIndex', idx.name
assert idx.hash_key == 'department', idx.hash_key
assert idx.range_key == 'score', idx.range_key
args = DynamoQuery.query_arguments(table, q, index=idx)
assert args == {'score__gte': 25, 'department__eq': 'sales'} # age is not indexed, not in query filters
res = list(self.ds.query(q))
assert res == [tom]
# Filter on hash and secondary index key
q = Query(pkey).filter('department', '=', 'sales').filter('score','>',500)
res = list(self.ds.query(q))
assert res == [tom, johnny] or res == [johnny, tom]
# Filter on global hash key exclusively
q = Query(pkey).filter('group', '=', 'managers')
idx = DynamoQuery.index_for_query(table, q)
assert idx.name == 'GroupIndex', idx.name
assert idx.hash_key == 'group', idx.hash_key
assert idx.range_key == 'age', idx.range_key
res = list(self.ds.query(q))
assert res == [johnny, barbara] or res == [barbara, johnny], res
# Filter on global hash and range key
q = Query(pkey).filter('group', '=', 'managers').filter('age','=',40)
res = list(self.ds.query(q))
assert res == [barbara]
# Filter on global hash and range key with comparison
q = Query(pkey).filter('group', '=', 'managers').filter('age','<',30)
res = list(self.ds.query(q))
assert res == [johnny]
# Filter on global hash key and an arbitrary other non-indexed key
q = Query(pkey).filter('group', '=', 'managers').filter('score','>',500)
res = list(self.ds.query(q))
assert res == [johnny]
# Update a secondary index range key
tom['score'] = 400
self.ds.put(tom_key, tom)
# Query using secondary index again, verify updated values
q = Query(pkey).filter('department', '=', 'sales').filter('score','>',500)
res = list(self.ds.query(q))
assert res == [johnny] or res == [johnny]
# Update a global index rang key
johnny['age'] = 35
self.ds.put(johnny_key, johnny)
# Query using global index again, verify updated values
q = Query(pkey).filter('group', '=', 'managers').filter('age','>',30)
res = list(self.ds.query(q))
assert res == [johnny, barbara] or res == [barbara, johnny]
assert not mock_method.called
# Try to create an item with the key separator in the hash
bad_key = pkey.child('datastore.dynamo.Bad')
bad_item = {
'key': str(bad_key),
'department': 'datastore.dynamo',
'name': 'Bad',
'group': 'employees',
'age': 30,
'score': 1000
}
try:
self.ds.put(bad_key, bad_item)
assert False, "should have not allowed, bad key"
except ValueError:
pass
def test_symlink_internals(self):
from datastore.core.basic import SymlinkDatastore
dds = DictDatastore()
sds = SymlinkDatastore(dds)
a = Key('/A')
b = Key('/B')
c = Key('/C')
d = Key('/D')
lva = sds._link_value_for_key(a)
lvb = sds._link_value_for_key(b)
lvc = sds._link_value_for_key(c)
lvd = sds._link_value_for_key(d)
# helper to check queries
sds_query = lambda: list(sds.query(Query(Key('/'))))
dds_query = lambda: list(dds.query(Query(Key('/'))))
# ensure _link_value_for_key and _link_for_value work
self.assertEqual(lva, str(a.child(sds.sentinel)))
self.assertEqual(a, sds._link_for_value(lva))
# adding a value should work like usual
sds.put(a, 1)
self.assertEqual(sds.get(a), 1)
self.assertEqual(sds.get(b), None)
self.assertNotEqual(sds.get(b), sds.get(a))
self.assertEqual(dds.get(a), 1)
self.assertEqual(dds.get(b), None)
self.assertEqual(sds_query(), [1])
self.assertEqual(dds_query(), [1])
# _follow_link(sds._link_value_for_key(a)) should == get(a)
self.assertEqual(sds._follow_link(lva), 1)
self.assertEqual(list(sds._follow_link_gen([lva])), [1])
# linking keys should work
sds.link(a, b)
self.assertEqual(sds.get(a), 1)
self.assertEqual(sds.get(b), 1)
self.assertEqual(sds.get(a), sds.get(b))
self.assertEqual(dds.get(a), 1)
self.assertEqual(dds.get(b), lva)
self.assertEqual(sds_query(), [1, 1])
self.assertEqual(dds_query(), [lva, 1])
# changing link should affect source
sds.put(b, 2)
self.assertEqual(sds.get(a), 2)
self.assertEqual(sds.get(b), 2)
self.assertEqual(sds.get(a), sds.get(b))
self.assertEqual(dds.get(a), 2)
self.assertEqual(dds.get(b), lva)
self.assertEqual(sds_query(), [2, 2])
self.assertEqual(dds_query(), [lva, 2])
# deleting source should affect link
sds.delete(a)
self.assertEqual(sds.get(a), None)
self.assertEqual(sds.get(b), None)
self.assertEqual(sds.get(b), sds.get(a))
self.assertEqual(dds.get(a), None)
self.assertEqual(dds.get(b), lva)
self.assertEqual(sds_query(), [None])
self.assertEqual(dds_query(), [lva])
# putting back source should yield working link
sds.put(a, 3)
self.assertEqual(sds.get(a), 3)
self.assertEqual(sds.get(b), 3)
self.assertEqual(sds.get(b), sds.get(a))
self.assertEqual(dds.get(a), 3)
self.assertEqual(dds.get(b), lva)
self.assertEqual(sds_query(), [3, 3])
self.assertEqual(dds_query(), [lva, 3])
# deleting link should not affect source
sds.delete(b)
self.assertEqual(sds.get(a), 3)
self.assertEqual(sds.get(b), None)
self.assertNotEqual(sds.get(b), sds.get(a))
self.assertEqual(dds.get(a), 3)
self.assertEqual(dds.get(b), None)
self.assertEqual(sds_query(), [3])
self.assertEqual(dds_query(), [3])
# linking should bring back to normal
sds.link(a, b)
self.assertEqual(sds.get(a), 3)
self.assertEqual(sds.get(b), 3)
self.assertEqual(sds.get(b), sds.get(a))
self.assertEqual(dds.get(a), 3)
self.assertEqual(dds.get(b), lva)
self.assertEqual(sds_query(), [3, 3])
self.assertEqual(dds_query(), [lva, 3])
# Adding another link should not affect things.
sds.link(a, c)
self.assertEqual(sds.get(a), 3)
self.assertEqual(sds.get(b), 3)
self.assertEqual(sds.get(c), 3)
self.assertEqual(sds.get(a), sds.get(b))
self.assertEqual(sds.get(a), sds.get(c))
self.assertEqual(dds.get(a), 3)
self.assertEqual(dds.get(b), lva)
self.assertEqual(dds.get(c), lva)
self.assertEqual(sds_query(), [3, 3, 3])
self.assertEqual(dds_query(), [lva, lva, 3])
# linking should be transitive
sds.link(b, c)
sds.link(c, d)
self.assertEqual(sds.get(a), 3)
self.assertEqual(sds.get(b), 3)
self.assertEqual(sds.get(c), 3)
self.assertEqual(sds.get(d), 3)
self.assertEqual(sds.get(a), sds.get(b))
self.assertEqual(sds.get(a), sds.get(c))
self.assertEqual(sds.get(a), sds.get(d))
self.assertEqual(dds.get(a), 3)
self.assertEqual(dds.get(b), lva)
self.assertEqual(dds.get(c), lvb)
self.assertEqual(dds.get(d), lvc)
self.assertEqual(sds_query(), [3, 3, 3, 3])
self.assertEqual(set(dds_query()), set([3, lva, lvb, lvc]))
self.assertRaises(AssertionError, sds.link, d, a)
