def test_lowercase_transform(self):
from datastore.core.basic import KeyTransformDatastore
def transform(key):
return Key(str(key).lower())
ds = DictDatastore()
lds = KeyTransformDatastore(ds, keytransform=transform)
k1 = Key('hello')
k2 = Key('HELLO')
k3 = Key('HeLlo')
ds.put(k1, 'world')
ds.put(k2, 'WORLD')
self.assertEqual(ds.get(k1), 'world')
self.assertEqual(ds.get(k2), 'WORLD')
self.assertFalse(ds.contains(k3))
self.assertEqual(lds.get(k1), 'world')
self.assertEqual(lds.get(k2), 'world')
self.assertEqual(lds.get(k3), 'world')
def test(key, val):
lds.put(key, val)
self.assertEqual(lds.get(k1), val)
self.assertEqual(lds.get(k2), val)
self.assertEqual(lds.get(k3), val)
test(k1, 'a')
test(k2, 'b')
test(k3, 'c')
def test_namespace(self):
from datastore.core.basic import NamespaceDatastore
k1 = Key('/c/d')
k2 = Key('/a/b')
k3 = Key('/a/b/c/d')
ds = DictDatastore()
nd = NamespaceDatastore(k2, ds)
ds.put(k1, 'cd')
ds.put(k3, 'abcd')
self.assertEqual(ds.get(k1), 'cd')
self.assertFalse(ds.contains(k2))
self.assertEqual(ds.get(k3), 'abcd')
self.assertEqual(nd.get(k1), 'abcd')
self.assertFalse(nd.contains(k2))
self.assertFalse(nd.contains(k3))
def test(key, val):
nd.put(key, val)
self.assertEqual(nd.get(key), val)
self.assertFalse(ds.contains(key))
self.assertFalse(nd.contains(k2.child(key)))
self.assertEqual(ds.get(k2.child(key)), val)
for i in range(0, 10):
test(Key(str(i)), 'val%d' % i)
def test_directory_simple(self):
from datastore.core.basic import DirectoryDatastore
ds = DirectoryDatastore(DictDatastore())
# initialize directory at /foo
dir_key = Key('/foo')
ds.directory(dir_key)
# adding directory entries
bar_key = Key('/foo/bar')
baz_key = Key('/foo/baz')
ds.directoryAdd(dir_key, bar_key)
ds.directoryAdd(dir_key, baz_key)
keys = list(ds.directoryRead(dir_key))
self.assertEqual(keys, [bar_key, baz_key])
# removing directory entries
ds.directoryRemove(dir_key, bar_key)
keys = list(ds.directoryRead(dir_key))
self.assertEqual(keys, [baz_key])
ds.directoryRemove(dir_key, baz_key)
keys = list(ds.directoryRead(dir_key))
self.assertEqual(keys, [])
# generator
with self.assertRaises(StopIteration):
gen = ds.directoryRead(dir_key)
gen.next()
def test_lowercase(self):
from datastore.core.basic import LowercaseKeyDatastore
ds = DictDatastore()
lds = LowercaseKeyDatastore(ds)
k1 = Key('hello')
k2 = Key('HELLO')
k3 = Key('HeLlo')
ds.put(k1, 'world')
ds.put(k2, 'WORLD')
self.assertEqual(ds.get(k1), 'world')
self.assertEqual(ds.get(k2), 'WORLD')
self.assertFalse(ds.contains(k3))
self.assertEqual(lds.get(k1), 'world')
self.assertEqual(lds.get(k2), 'world')
self.assertEqual(lds.get(k3), 'world')
def test(key, val):
lds.put(key, val)
self.assertEqual(lds.get(k1), val)
self.assertEqual(lds.get(k2), val)
self.assertEqual(lds.get(k3), val)
test(k1, 'a')
test(k2, 'b')
test(k3, 'c')
def test_type(self):
k1 = Key('/A/B/C:c')
k2 = Key('/A/B/C:c/D:d')
assert k1.is_ancestor_of(k2)
assert k2.is_descendant_of(k1)
assert k1.type == 'C'
assert k2.type == 'D'
assert k1.type == k2.parent.type
def test_simple(self):
from datastore.core.basic import NamespaceDatastore
s1 = NamespaceDatastore(Key('a'), DictDatastore())
s2 = NamespaceDatastore(Key('b'), DictDatastore())
s3 = NamespaceDatastore(Key('c'), DictDatastore())
stores = [s1, s2, s3]
self.subtest_simple(stores)
def test_5_3(self):
opts = {}
opts['k1'] = Key('/abcdefghijk')
opts['k2'] = Key('/abcdefghijki')
opts['k3'] = Key('/abc/def/ghi/jka/bcd/abcdefghijk')
opts['k4'] = Key('/abc/def/ghi/jki/abc/abcdefghijki')
opts['depth'] = 5
opts['length'] = 3
self.subtest_nested_path_ds(**opts)
def test_3_2(self):
opts = {}
opts['k1'] = Key('/abcdefghijk')
opts['k2'] = Key('/abcdefghijki')
opts['k3'] = Key('/ab/cd/ef/abcdefghijk')
opts['k4'] = Key('/ab/cd/ef/abcdefghijki')
opts['depth'] = 3
opts['length'] = 2
self.subtest_nested_path_ds(**opts)
def test_type(self):
k1 = Key('/A/B/C:c')
k2 = Key('/A/B/C:c/D:d')
self.assertRaises(TypeError, k1.is_ancestor_of, str(k2))
self.assertTrue(k1.is_ancestor_of(k2))
self.assertTrue(k2.is_descendant_of(k1))
self.assertEqual(k1.type, 'C')
self.assertEqual(k2.type, 'D')
self.assertEqual(k1.type, k2.parent.type)
def test_cursor(self):
k = Key('/')
with pytest.raises(ValueError):
Cursor(None, None)
with pytest.raises(ValueError):
Cursor(Query(Key('/')), None)
with pytest.raises(ValueError):
Cursor(None, [1])
c = Cursor(Query(k), [1, 2, 3, 4, 5]) # should not raise
assert c.skipped == 0
assert c.returned == 0
assert c._iterable == [1, 2, 3, 4, 5]
c.skipped = 1
c.returned = 2
assert c.skipped == 1
assert c.returned == 2
c._skipped_inc(None)
c._skipped_inc(None)
assert c.skipped == 3
c._returned_inc(None)
c._returned_inc(None)
c._returned_inc(None)
assert c.returned == 5
self.subtest_cursor(Query(k), [5, 4, 3, 2, 1], [5, 4, 3, 2, 1])
self.subtest_cursor(Query(k, limit=3), [5, 4, 3, 2, 1], [5, 4, 3])
self.subtest_cursor(Query(k, limit=0), [5, 4, 3, 2, 1], [])
self.subtest_cursor(Query(k, offset=2), [5, 4, 3, 2, 1], [3, 2, 1])
self.subtest_cursor(Query(k, offset=5), [5, 4, 3, 2, 1], [])
self.subtest_cursor(Query(k, limit=2, offset=2), [5, 4, 3, 2, 1],
[3, 2])
v1, v2, v3 = version_objects()
vs = [v1, v2, v3]
t1 = v1['committed']
t2 = v2['committed']
self.subtest_cursor(Query(k), vs, vs)
self.subtest_cursor(Query(k, limit=2), vs, [v1, v2])
self.subtest_cursor(Query(k, offset=1), vs, [v2, v3])
self.subtest_cursor(Query(k, offset=1, limit=1), vs, [v2])
self.subtest_cursor(
Query(k).filter('committed', '>=', t2), vs, [v2, v3])
self.subtest_cursor(Query(k).filter('committed', '<=', t1), vs, [v1])
self.subtest_cursor(Query(k).order('+committed'), vs, [v1, v2, v3])
self.subtest_cursor(Query(k).order('-created'), vs, [v3, v2, v1])
def test_type(self):
k1 = Key('/A/B/C:c')
k2 = Key('/A/B/C:c/D:d')
with pytest.raises(TypeError):
k1.is_ancestor_of(str(k2))
assert k1.is_ancestor_of(k2)
assert k2.is_descendant_of(k1)
assert k1.type == 'C'
assert k2.type == 'D'
assert k1.type == k2.parent.type
def test_keyfn(self):
opts = {}
opts['k1'] = Key('/abcdefghijk')
opts['k2'] = Key('/abcdefghijki')
opts['k3'] = Key('/kj/ih/gf/abcdefghijk')
opts['k4'] = Key('/ik/ji/hg/abcdefghijki')
opts['depth'] = 3
opts['length'] = 2
opts['key_fn'] = lambda key: key.name[::-1]
self.subtest_nested_path_ds(**opts)
def test_symlink_recursive(self):
from datastore.core.basic import SymlinkDatastore
dds = DictDatastore()
sds1 = SymlinkDatastore(dds)
sds2 = SymlinkDatastore(sds1)
a = Key('/A')
b = Key('/B')
sds2.put(a, 1)
self.assertEqual(sds2.get(a), 1)
self.assertEqual(sds2.get(b), None)
self.assertNotEqual(sds2.get(b), sds2.get(a))
sds2.link(a, b)
self.assertEqual(sds2.get(a), 1)
self.assertEqual(sds2.get(b), 1)
self.assertEqual(sds2.get(a), sds2.get(b))
self.assertEqual(sds1.get(a), sds1.get(b))
sds2.link(a, b)
self.assertEqual(sds2.get(a), 1)
self.assertEqual(sds2.get(b), 1)
self.assertEqual(sds2.get(a), sds2.get(b))
self.assertEqual(sds1.get(a), sds1.get(b))
sds2.link(a, b)
self.assertEqual(sds2.get(a), 1)
self.assertEqual(sds2.get(b), 1)
self.assertEqual(sds2.get(a), sds2.get(b))
self.assertEqual(sds1.get(a), sds1.get(b))
sds2.put(b, 2)
self.assertEqual(sds2.get(a), 2)
self.assertEqual(sds2.get(b), 2)
self.assertEqual(sds2.get(a), sds2.get(b))
self.assertEqual(sds1.get(a), sds1.get(b))
sds2.delete(a)
self.assertEqual(sds2.get(a), None)
self.assertEqual(sds2.get(b), None)
self.assertEqual(sds2.get(b), sds2.get(a))
sds2.put(a, 3)
self.assertEqual(sds2.get(a), 3)
self.assertEqual(sds2.get(b), 3)
self.assertEqual(sds2.get(b), sds2.get(a))
sds2.delete(b)
self.assertEqual(sds2.get(a), 3)
self.assertEqual(sds2.get(b), None)
self.assertNotEqual(sds2.get(b), sds2.get(a))
def test_cursor(self):
k = Key('/')
self.assertRaises(ValueError, Cursor, None, None)
self.assertRaises(ValueError, Cursor, Query(Key('/')), None)
self.assertRaises(ValueError, Cursor, None, [1])
c = Cursor(Query(k), [1, 2, 3, 4, 5]) # should not raise
self.assertEqual(c.skipped, 0)
self.assertEqual(c.returned, 0)
self.assertEqual(c._iterable, [1, 2, 3, 4, 5])
c.skipped = 1
c.returned = 2
self.assertEqual(c.skipped, 1)
self.assertEqual(c.returned, 2)
c._skipped_inc(None)
c._skipped_inc(None)
self.assertEqual(c.skipped, 3)
c._returned_inc(None)
c._returned_inc(None)
c._returned_inc(None)
self.assertEqual(c.returned, 5)
self.subtest_cursor(Query(k), [5, 4, 3, 2, 1], [5, 4, 3, 2, 1])
self.subtest_cursor(Query(k, limit=3), [5, 4, 3, 2, 1], [5, 4, 3])
self.subtest_cursor(Query(k, limit=0), [5, 4, 3, 2, 1], [])
self.subtest_cursor(Query(k, offset=2), [5, 4, 3, 2, 1], [3, 2, 1])
self.subtest_cursor(Query(k, offset=5), [5, 4, 3, 2, 1], [])
self.subtest_cursor(Query(k, limit=2, offset=2), [5, 4, 3, 2, 1],
[3, 2])
v1, v2, v3 = version_objects()
vs = [v1, v2, v3]
t1 = v1['committed']
t2 = v2['committed']
t3 = v3['committed']
self.subtest_cursor(Query(k), vs, vs)
self.subtest_cursor(Query(k, limit=2), vs, [v1, v2])
self.subtest_cursor(Query(k, offset=1), vs, [v2, v3])
self.subtest_cursor(Query(k, offset=1, limit=1), vs, [v2])
self.subtest_cursor(
Query(k).filter('committed', '>=', t2), vs, [v2, v3])
self.subtest_cursor(Query(k).filter('committed', '<=', t1), vs, [v1])
self.subtest_cursor(Query(k).order('+committed'), vs, [v1, v2, v3])
self.subtest_cursor(Query(k).order('-created'), vs, [v3, v2, v1])
def test_null(self):
from datastore.core.basic import NullDatastore
s = NullDatastore()
for c in range(1, 20):
c = str(c)
k = Key(c)
self.assertFalse(s.contains(k))
self.assertEqual(s.get(k), None)
s.put(k, c)
self.assertFalse(s.contains(k))
self.assertEqual(s.get(k), None)
for item in s.query(Query(Key('/'))):
raise Exception('Should not have found anything.')
def _link_for_value(self, value):
"""Returns the linked key if `value` is a link, or None."""
try:
key = Key(value)
if key.name == self.sentinel:
return key.parent
except:
pass
return None
def test_basic(self):
now = time.time_ns()
q1 = Query(Key('/'), limit=100)
q2 = Query(Key('/'), offset=200)
q3 = Query(Key('/'), object_getattr=getattr)
q1.offset = 300
q3.limit = 1
q1.filter('key', '>', '/ABC')
q1.filter('created', '>', now)
q2.order('key')
q2.order('-created')
q1d = {
'key': '/',
'limit': 100,
'offset': 300,
'filter': [['key', '>', '/ABC'], ['created', '>', now]]
}
q2d = {'key': '/', 'offset': 200, 'order': ['+key', '-created']}
q3d = {'key': '/', 'limit': 1}
assert q1.dict() == q1d
assert q2.dict() == q2d
assert q3.dict() == q3d
assert q1 == Query.from_dict(q1d)
assert q2 == Query.from_dict(q2d)
assert q3 == Query.from_dict(q3d)
assert q1 == eval(repr(q1))
assert q2 == eval(repr(q2))
assert q3 == eval(repr(q3))
assert q1 == q1.copy()
assert q2 == q2.copy()
assert q3 == q3.copy()
def test_directory_init(self):
from datastore.core.basic import DirectoryDatastore
ds = DirectoryDatastore(DictDatastore())
# initialize directory at /foo
dir_key = Key('/foo')
ds.directory(dir_key)
self.assertEqual(ds.get(dir_key), [])
# can add to dir
bar_key = Key('/foo/bar')
ds.directoryAdd(dir_key, bar_key)
self.assertEqual(ds.get(dir_key), [str(bar_key)])
# re-init does not wipe out directory at /foo
dir_key = Key('/foo')
ds.directory(dir_key)
self.assertEqual(ds.get(dir_key), [str(bar_key)])
def nestKey(self, key):
"""Returns a nested `key`."""
nest = self.nest_keyfn(key)
# if depth * length > len(key.name), we need to pad.
mult = 1 + int(self.nest_depth * self.nest_length / len(nest))
nest = nest * mult
pref = Key(self.nestedPath(nest, self.nest_depth, self.nest_length))
return pref.child(key)
def test_ancestry(self):
k1 = Key('/A/B/C')
k2 = Key('/A/B/C/D')
assert k1._string == '/A/B/C'
assert k2._string == '/A/B/C/D'
assert k1.is_ancestor_of(k2)
assert k2.is_descendant_of(k1)
assert Key('/A').is_ancestor_of(k2)
assert Key('/A').is_ancestor_of(k1)
assert not Key('/A').is_descendant_of(k2)
assert not Key('/A').is_descendant_of(k1)
assert k2.is_descendant_of(Key('/A'))
assert k1.is_descendant_of(Key('/A'))
assert not k2.is_ancestor_of(Key('/A'))
assert not k1.is_ancestor_of(Key('/A'))
assert not k2.is_ancestor_of(k2)
assert not k1.is_ancestor_of(k1)
assert k1.child('D') == k2
assert k1 == k2.parent
assert k1.path == k2.parent.path
def test_ancestry(self):
k1 = Key('/A/B/C')
k2 = Key('/A/B/C/D')
self.assertEqual(k1._string, '/A/B/C')
self.assertEqual(k2._string, '/A/B/C/D')
self.assertTrue(k1.is_ancestor_of(k2))
self.assertTrue(k2.is_descendant_of(k1))
self.assertTrue(Key('/A').is_ancestor_of(k2))
self.assertTrue(Key('/A').is_ancestor_of(k1))
self.assertFalse(Key('/A').is_descendant_of(k2))
self.assertFalse(Key('/A').is_descendant_of(k1))
self.assertTrue(k2.is_descendant_of(Key('/A')))
self.assertTrue(k1.is_descendant_of(Key('/A')))
self.assertFalse(k2.is_ancestor_of(Key('/A')))
self.assertFalse(k1.is_ancestor_of(Key('/A')))
self.assertFalse(k2.is_ancestor_of(k2))
self.assertFalse(k1.is_ancestor_of(k1))
self.assertEqual(k1.child('D'), k2)
self.assertEqual(k1, k2.parent)
self.assertEqual(k1.path, k2.parent.path)
def test_directory_double_add(self):
from datastore.core.basic import DirectoryDatastore
ds = DirectoryDatastore(DictDatastore())
# initialize directory at /foo
dir_key = Key('/foo')
ds.directory(dir_key)
# adding directory entries
bar_key = Key('/foo/bar')
baz_key = Key('/foo/baz')
ds.directoryAdd(dir_key, bar_key)
ds.directoryAdd(dir_key, baz_key)
ds.directoryAdd(dir_key, bar_key)
ds.directoryAdd(dir_key, baz_key)
ds.directoryAdd(dir_key, baz_key)
ds.directoryAdd(dir_key, bar_key)
keys = list(ds.directoryRead(dir_key))
self.assertEqual(keys, [bar_key, baz_key])
def test_basic(self):
now = nanotime.now().nanoseconds()
q1 = Query(Key('/'), limit=100)
q2 = Query(Key('/'), offset=200)
q3 = Query(Key('/'), object_getattr=getattr)
q1.offset = 300
q3.limit = 1
q1.filter('key', '>', '/ABC')
q1.filter('created', '>', now)
q2.order('key')
q2.order('-created')
q1d = {'key': '/', 'limit': 100, 'offset': 300, \
'filter': [['key', '>', '/ABC'], ['created', '>', now]]}
q2d = {'key': '/', 'offset': 200, 'order': ['+key', '-created']}
q3d = {'key': '/', 'limit': 1}
self.assertEqual(q1.dict(), q1d)
self.assertEqual(q2.dict(), q2d)
self.assertEqual(q3.dict(), q3d)
self.assertEqual(q1, Query.from_dict(q1d))
self.assertEqual(q2, Query.from_dict(q2d))
self.assertEqual(q3, Query.from_dict(q3d))
self.assertEqual(q1, eval(repr(q1)))
self.assertEqual(q2, eval(repr(q2)))
self.assertEqual(q3, eval(repr(q3)))
self.assertEqual(q1, q1.copy())
self.assertEqual(q2, q2.copy())
self.assertEqual(q3, q3.copy())
def test_reverse_transform(self):
from datastore.core.basic import KeyTransformDatastore
def transform(key):
return key.reverse
ds = DictDatastore()
kt = KeyTransformDatastore(ds, keytransform=transform)
k1 = Key('/a/b/c')
k2 = Key('/c/b/a')
self.assertFalse(ds.contains(k1))
self.assertFalse(ds.contains(k2))
self.assertFalse(kt.contains(k1))
self.assertFalse(kt.contains(k2))
ds.put(k1, 'abc')
self.assertEqual(ds.get(k1), 'abc')
self.assertFalse(ds.contains(k2))
self.assertFalse(kt.contains(k1))
self.assertEqual(kt.get(k2), 'abc')
kt.put(k1, 'abc')
self.assertEqual(ds.get(k1), 'abc')
self.assertEqual(ds.get(k2), 'abc')
self.assertEqual(kt.get(k1), 'abc')
self.assertEqual(kt.get(k2), 'abc')
ds.delete(k1)
self.assertFalse(ds.contains(k1))
self.assertEqual(ds.get(k2), 'abc')
self.assertEqual(kt.get(k1), 'abc')
self.assertFalse(kt.contains(k2))
kt.delete(k1)
self.assertFalse(ds.contains(k1))
self.assertFalse(ds.contains(k2))
self.assertFalse(kt.contains(k1))
self.assertFalse(kt.contains(k2))
def subtest_serializer_shim(self, serializer, numelems=100):
child = DictDatastore()
shim = SerializerShimDatastore(child, serializer=serializer)
values_raw = [{'value': i} for i in range(0, numelems)]
values_serial = [serializer.dumps(v) for v in values_raw]
values_deserial = [serializer.loads(v) for v in values_serial]
self.assertEqual(values_deserial, values_raw)
for value in values_raw:
key = Key(value['value'])
value_serialized = serializer.dumps(value)
# should not be there yet
self.assertFalse(shim.contains(key))
self.assertEqual(shim.get(key), None)
# put (should be there)
shim.put(key, value)
self.assertTrue(shim.contains(key))
self.assertEqual(shim.get(key), value)
# make sure underlying DictDatastore is storing the serialized value.
self.assertEqual(shim.child_datastore.get(key), value_serialized)
# delete (should not be there)
shim.delete(key)
self.assertFalse(shim.contains(key))
self.assertEqual(shim.get(key), None)
# make sure manipulating underlying DictDatastore works equally well.
shim.child_datastore.put(key, value_serialized)
self.assertTrue(shim.contains(key))
self.assertEqual(shim.get(key), value)
shim.child_datastore.delete(key)
self.assertFalse(shim.contains(key))
self.assertEqual(shim.get(key), None)
if serializer is not bson: # bson can't handle non mapping types
self.subtest_simple([shim], numelems)
def from_dict(cls, dictionary):
"""Constructs a query from a dictionary."""
query = cls(Key(dictionary['key']))
for key, value in dictionary.items():
if key == 'order':
for order in value:
query.order(order)
elif key == 'filter':
for filter in value:
if not isinstance(filter, Filter):
filter = Filter(*filter)
query.filter(filter)
elif key in ['limit', 'offset', 'offset_key']:
setattr(query, key, value)
return query
def __init__(self,
key=Key('/'),
limit=None,
offset=0,
offset_key=None,
object_getattr=None):
""" Initialize a query.
Parameters
key: a key representing the level of this query. For example, a Query with
Key('/MontyPython/Actor:') would query objects in that key path, eg:
Key('/MontyPython/Actor:JohnCleese')
Key('/MontyPython/Actor:EricIdle')
Key('/MontyPython/Actor:GrahamChapman')
It is up to datastores how to implement this namespacing. E.g.,
some datastores may store values in different tables or collections.
limit: an integer representing the maximum number of results to return.
offset: an integer representing a number of results to skip.
object_getattr: a function to extract attribute values from an object. It
is used to satisfy query filters and orders. Defining this function
allows the client to control the data model of the stored values.
The default function attempts to access values as attributes
(__getattr__) or items (__getitem__).
"""
if not isinstance(key, Key):
raise TypeError('key must be of type %s' % Key)
self.key = key
self.limit = int(limit) if limit is not None else None
self.offset = int(offset)
self.offset_key = offset_key
self.filters = []
self.orders = []
if object_getattr:
self.object_getattr = object_getattr
def directory_values_generator(self, key):
"""Retrieve directory values for given key."""
directory = self.directory(key)
for key in directory:
yield self.get(Key(key))
def directory_entries_generator(self, dir_key):
dir_items = self.get(dir_key) or []
for item in dir_items:
yield Key(item)
def __init__(self, namespace, *args, **kwargs):
"""Initializes NamespaceDatastore with `key` namespace."""
super(NamespaceDatastore, self).__init__(*args, **kwargs)
self.keytransform = self.namespaceKey
self.namespace = Key(namespace)
