def check_query(self, query, total, slice):
allitems = list(range(0, total))
resultset = None
for sn in self.stores:
try:
contents = list(sn.query(Query(self.pkey)))
expected = contents[slice]
resultset = sn.query(query)
result = list(resultset)
# make sure everything is there.
self.assertTrue(len(contents) == len(allitems), \
'%s == %s' % (str(contents), str(allitems)))
self.assertTrue(all([val in contents for val in allitems]))
self.assertTrue(len(result) == len(expected), \
'%s == %s' % (str(result), str(expected)))
self.assertTrue(all([val in result for val in expected]))
# TODO: should order be preserved?
# self.assertEqual(result, expected)
except NotImplementedError:
print
'WARNING: %s does not implement query.' % sn
return resultset
def subtest_queries(self):
for value in range(0, self.numelems):
key = self.pkey.child(value)
for sn in self.stores:
sn.put(key, value)
k = self.pkey
n = int(self.numelems)
self.check_query(Query(k), n, slice(0, n))
self.check_query(Query(k, limit=n), n, slice(0, n))
self.check_query(Query(k, limit=n / 2), n, slice(0, n / 2))
self.check_query(Query(k, offset=n / 2), n, slice(n / 2, n))
self.check_query(Query(k, offset=n / 3, limit=n / 3), n,
slice(n / 3, 2 * (n / 3)))
del k
del n
def test_null(self):
from datastore.core.basic import NullDatastore
s = NullDatastore()
for c in range(1, 20):
c = str(c)
k = Key(c)
self.assertFalse(s.contains(k))
self.assertEqual(s.get(k), None)
s.put(k, c)
self.assertFalse(s.contains(k))
self.assertEqual(s.get(k), None)
for item in s.query(Query(Key('/'))):
raise Exception('Should not have found anything.')
def test_symlink_internals(self):
from datastore.core.basic import SymlinkDatastore
dds = DictDatastore()
sds = SymlinkDatastore(dds)
a = Key('/A')
b = Key('/B')
c = Key('/C')
d = Key('/D')
lva = sds._link_value_for_key(a)
lvb = sds._link_value_for_key(b)
lvc = sds._link_value_for_key(c)
lvd = sds._link_value_for_key(d)
# helper to check queries
sds_query = lambda: list(sds.query(Query(Key('/'))))
dds_query = lambda: list(dds.query(Query(Key('/'))))
# ensure _link_value_for_key and _link_for_value work
self.assertEqual(lva, str(a.child(sds.sentinel)))
self.assertEqual(a, sds._link_for_value(lva))
# adding a value should work like usual
sds.put(a, 1)
self.assertEqual(sds.get(a), 1)
self.assertEqual(sds.get(b), None)
self.assertNotEqual(sds.get(b), sds.get(a))
self.assertEqual(dds.get(a), 1)
self.assertEqual(dds.get(b), None)
self.assertEqual(sds_query(), [1])
self.assertEqual(dds_query(), [1])
# _follow_link(sds._link_value_for_key(a)) should == get(a)
self.assertEqual(sds._follow_link(lva), 1)
self.assertEqual(list(sds._follow_link_gen([lva])), [1])
# linking keys should work
sds.link(a, b)
self.assertEqual(sds.get(a), 1)
self.assertEqual(sds.get(b), 1)
self.assertEqual(sds.get(a), sds.get(b))
self.assertEqual(dds.get(a), 1)
self.assertEqual(dds.get(b), lva)
self.assertEqual(sds_query(), [1, 1])
self.assertEqual(dds_query(), [lva, 1])
# changing link should affect source
sds.put(b, 2)
self.assertEqual(sds.get(a), 2)
self.assertEqual(sds.get(b), 2)
self.assertEqual(sds.get(a), sds.get(b))
self.assertEqual(dds.get(a), 2)
self.assertEqual(dds.get(b), lva)
self.assertEqual(sds_query(), [2, 2])
self.assertEqual(dds_query(), [lva, 2])
# deleting source should affect link
sds.delete(a)
self.assertEqual(sds.get(a), None)
self.assertEqual(sds.get(b), None)
self.assertEqual(sds.get(b), sds.get(a))
self.assertEqual(dds.get(a), None)
self.assertEqual(dds.get(b), lva)
self.assertEqual(sds_query(), [None])
self.assertEqual(dds_query(), [lva])
# putting back source should yield working link
sds.put(a, 3)
self.assertEqual(sds.get(a), 3)
self.assertEqual(sds.get(b), 3)
self.assertEqual(sds.get(b), sds.get(a))
self.assertEqual(dds.get(a), 3)
self.assertEqual(dds.get(b), lva)
self.assertEqual(sds_query(), [3, 3])
self.assertEqual(dds_query(), [lva, 3])
# deleting link should not affect source
sds.delete(b)
self.assertEqual(sds.get(a), 3)
self.assertEqual(sds.get(b), None)
self.assertNotEqual(sds.get(b), sds.get(a))
self.assertEqual(dds.get(a), 3)
self.assertEqual(dds.get(b), None)
self.assertEqual(sds_query(), [3])
self.assertEqual(dds_query(), [3])
# linking should bring back to normal
sds.link(a, b)
self.assertEqual(sds.get(a), 3)
self.assertEqual(sds.get(b), 3)
self.assertEqual(sds.get(b), sds.get(a))
self.assertEqual(dds.get(a), 3)
self.assertEqual(dds.get(b), lva)
self.assertEqual(sds_query(), [3, 3])
self.assertEqual(dds_query(), [lva, 3])
# Adding another link should not affect things.
sds.link(a, c)
self.assertEqual(sds.get(a), 3)
self.assertEqual(sds.get(b), 3)
self.assertEqual(sds.get(c), 3)
self.assertEqual(sds.get(a), sds.get(b))
self.assertEqual(sds.get(a), sds.get(c))
self.assertEqual(dds.get(a), 3)
self.assertEqual(dds.get(b), lva)
self.assertEqual(dds.get(c), lva)
self.assertEqual(sds_query(), [3, 3, 3])
self.assertEqual(dds_query(), [lva, lva, 3])
# linking should be transitive
sds.link(b, c)
sds.link(c, d)
self.assertEqual(sds.get(a), 3)
self.assertEqual(sds.get(b), 3)
self.assertEqual(sds.get(c), 3)
self.assertEqual(sds.get(d), 3)
self.assertEqual(sds.get(a), sds.get(b))
self.assertEqual(sds.get(a), sds.get(c))
self.assertEqual(sds.get(a), sds.get(d))
self.assertEqual(dds.get(a), 3)
self.assertEqual(dds.get(b), lva)
self.assertEqual(dds.get(c), lvb)
self.assertEqual(dds.get(d), lvc)
self.assertEqual(sds_query(), [3, 3, 3, 3])
self.assertEqual(set(dds_query()), set([3, lva, lvb, lvc]))
self.assertRaises(AssertionError, sds.link, d, a)
def test_cursor(self):
k = Key('/')
self.assertRaises(ValueError, Cursor, None, None)
self.assertRaises(ValueError, Cursor, Query(Key('/')), None)
self.assertRaises(ValueError, Cursor, None, [1])
c = Cursor(Query(k), [1, 2, 3, 4, 5]) # should not raise
self.assertEqual(c.skipped, 0)
self.assertEqual(c.returned, 0)
self.assertEqual(c._iterable, [1, 2, 3, 4, 5])
c.skipped = 1
c.returned = 2
self.assertEqual(c.skipped, 1)
self.assertEqual(c.returned, 2)
c._skipped_inc(None)
c._skipped_inc(None)
self.assertEqual(c.skipped, 3)
c._returned_inc(None)
c._returned_inc(None)
c._returned_inc(None)
self.assertEqual(c.returned, 5)
self.subtest_cursor(Query(k), [5, 4, 3, 2, 1], [5, 4, 3, 2, 1])
self.subtest_cursor(Query(k, limit=3), [5, 4, 3, 2, 1], [5, 4, 3])
self.subtest_cursor(Query(k, limit=0), [5, 4, 3, 2, 1], [])
self.subtest_cursor(Query(k, offset=2), [5, 4, 3, 2, 1], [3, 2, 1])
self.subtest_cursor(Query(k, offset=5), [5, 4, 3, 2, 1], [])
self.subtest_cursor(Query(k, limit=2, offset=2), [5, 4, 3, 2, 1],
[3, 2])
v1, v2, v3 = version_objects()
vs = [v1, v2, v3]
t1 = v1['committed']
t2 = v2['committed']
t3 = v3['committed']
self.subtest_cursor(Query(k), vs, vs)
self.subtest_cursor(Query(k, limit=2), vs, [v1, v2])
self.subtest_cursor(Query(k, offset=1), vs, [v2, v3])
self.subtest_cursor(Query(k, offset=1, limit=1), vs, [v2])
self.subtest_cursor(
Query(k).filter('committed', '>=', t2), vs, [v2, v3])
self.subtest_cursor(Query(k).filter('committed', '<=', t1), vs, [v1])
self.subtest_cursor(Query(k).order('+committed'), vs, [v1, v2, v3])
self.subtest_cursor(Query(k).order('-created'), vs, [v3, v2, v1])
def test_basic(self):
now = nanotime.now().nanoseconds()
q1 = Query(Key('/'), limit=100)
q2 = Query(Key('/'), offset=200)
q3 = Query(Key('/'), object_getattr=getattr)
q1.offset = 300
q3.limit = 1
q1.filter('key', '>', '/ABC')
q1.filter('created', '>', now)
q2.order('key')
q2.order('-created')
q1d = {'key': '/', 'limit': 100, 'offset': 300, \
'filter': [['key', '>', '/ABC'], ['created', '>', now]]}
q2d = {'key': '/', 'offset': 200, 'order': ['+key', '-created']}
q3d = {'key': '/', 'limit': 1}
self.assertEqual(q1.dict(), q1d)
self.assertEqual(q2.dict(), q2d)
self.assertEqual(q3.dict(), q3d)
self.assertEqual(q1, Query.from_dict(q1d))
self.assertEqual(q2, Query.from_dict(q2d))
self.assertEqual(q3, Query.from_dict(q3d))
self.assertEqual(q1, eval(repr(q1)))
self.assertEqual(q2, eval(repr(q2)))
self.assertEqual(q3, eval(repr(q3)))
self.assertEqual(q1, q1.copy())
self.assertEqual(q2, q2.copy())
self.assertEqual(q3, q3.copy())
def test_cursor(self):
k = Key('/')
with pytest.raises(ValueError):
Cursor(None, None)
with pytest.raises(ValueError):
Cursor(Query(Key('/')), None)
with pytest.raises(ValueError):
Cursor(None, [1])
c = Cursor(Query(k), [1, 2, 3, 4, 5]) # should not raise
assert c.skipped == 0
assert c.returned == 0
assert c._iterable == [1, 2, 3, 4, 5]
c.skipped = 1
c.returned = 2
assert c.skipped == 1
assert c.returned == 2
c._skipped_inc(None)
c._skipped_inc(None)
assert c.skipped == 3
c._returned_inc(None)
c._returned_inc(None)
c._returned_inc(None)
assert c.returned == 5
self.subtest_cursor(Query(k), [5, 4, 3, 2, 1], [5, 4, 3, 2, 1])
self.subtest_cursor(Query(k, limit=3), [5, 4, 3, 2, 1], [5, 4, 3])
self.subtest_cursor(Query(k, limit=0), [5, 4, 3, 2, 1], [])
self.subtest_cursor(Query(k, offset=2), [5, 4, 3, 2, 1], [3, 2, 1])
self.subtest_cursor(Query(k, offset=5), [5, 4, 3, 2, 1], [])
self.subtest_cursor(Query(k, limit=2, offset=2), [5, 4, 3, 2, 1], [3, 2])
v1, v2, v3 = version_objects()
vs = [v1, v2, v3]
t1 = v1['committed']
t2 = v2['committed']
self.subtest_cursor(Query(k), vs, vs)
self.subtest_cursor(Query(k, limit=2), vs, [v1, v2])
self.subtest_cursor(Query(k, offset=1), vs, [v2, v3])
self.subtest_cursor(Query(k, offset=1, limit=1), vs, [v2])
self.subtest_cursor(Query(k).filter('committed', '>=', t2), vs, [v2, v3])
self.subtest_cursor(Query(k).filter('committed', '<=', t1), vs, [v1])
self.subtest_cursor(Query(k).order('+committed'), vs, [v1, v2, v3])
self.subtest_cursor(Query(k).order('-created'), vs, [v3, v2, v1])
def test_basic(self):
now = time.time_ns()
q1 = Query(Key('/'), limit=100)
q2 = Query(Key('/'), offset=200)
q3 = Query(Key('/'), object_getattr=getattr)
q1.offset = 300
q3.limit = 1
q1.filter('key', '>', '/ABC')
q1.filter('created', '>', now)
q2.order('key')
q2.order('-created')
q1d = {'key': '/', 'limit': 100, 'offset': 300,
'filter': [['key', '>', '/ABC'], ['created', '>', now]]}
q2d = {'key': '/', 'offset': 200, 'order': ['+key', '-created']}
q3d = {'key': '/', 'limit': 1}
assert q1.dict() == q1d
assert q2.dict() == q2d
assert q3.dict() == q3d
assert q1 == Query.from_dict(q1d)
assert q2 == Query.from_dict(q2d)
assert q3 == Query.from_dict(q3d)
assert q1 == eval(repr(q1))
assert q2 == eval(repr(q2))
assert q3 == eval(repr(q3))
assert q1 == q1.copy()
assert q2 == q2.copy()
assert q3 == q3.copy()