def setUp(self):
identity = lambda x: x
to_int = lambda x: int(x)
to_float = lambda x: float(x)
to_bool = lambda x: bool(x)
schema = {
'pk': {
'convert': identity,
'index': None
}, #will be indexed anyways
'ts': {
'convert': identity,
'index': None
},
'order': {
'convert': to_int,
'index': 1
},
'blarg': {
'convert': to_int,
'index': 1
},
'useless': {
'convert': identity,
'index': None
},
'mean': {
'convert': to_float,
'index': 1
},
'std': {
'convert': to_float,
'index': 1
},
'vp': {
'convert': to_bool,
'index': 1
}
}
self.db = DictDB(schema, 'pk', 3)
self.db.insert_ts('one', ts.TimeSeries([1, 2, 3], [1, 4, 9]))
self.db.insert_ts('two', ts.TimeSeries([2, 3, 4], [4, 9, 16]))
self.db.insert_ts('three', ts.TimeSeries([9, 3, 4], [4, 0, 16]))
self.db.insert_ts('four', ts.TimeSeries([0, 0, 4], [1, 0, 4]))
self.db.upsert_meta('one', {'order': 1, 'blarg': 1})
self.db.upsert_meta('two', {'order': 2})
self.db.upsert_meta('three', {'order': 1, 'blarg': 2})
self.db.upsert_meta('four', {'order': 2, 'blarg': 2})
def setUp(self):
identity = lambda x: x
to_int = lambda x:int(x)
to_float = lambda x:float(x)
to_bool = lambda x:bool(x)
schema = {
'pk': {'convert': identity, 'index': None}, #will be indexed anyways
'ts': {'convert': identity, 'index': None},
'order': {'convert': to_int, 'index': 1},
'blarg': {'convert': to_int, 'index': 1},
'useless': {'convert': identity, 'index': None},
'mean': {'convert': to_float, 'index': 1},
'std': {'convert': to_float, 'index': 1},
'vp': {'convert': to_bool, 'index': 1}
}
self.db = DictDB(schema,'pk',3)
self.db.insert_ts('one',ts.TimeSeries([1, 2, 3],[1, 4, 9]))
self.db.insert_ts('two',ts.TimeSeries([2, 3, 4],[4, 9, 16]))
self.db.insert_ts('three',ts.TimeSeries([9,3,4],[4,0,16]))
self.db.insert_ts('four',ts.TimeSeries([0,0,4],[1,0,4]))
self.db.upsert_meta('one', {'order': 1, 'blarg': 1})
self.db.upsert_meta('two', {'order': 2})
self.db.upsert_meta('three', {'order': 1, 'blarg': 2})
self.db.upsert_meta('four', {'order': 2, 'blarg': 2})
def main():
# we augment the schema by adding columns for 5 vantage points
for i in range(NUMVPS):
schema["d_vp-{}".format(i)] = {'convert': float, 'index': 1}
db = DictDB(schema, 'pk')
server = TSDBServer(db)
server.run()
def test_complex():
db = DictDB(schema, 'pk')
for i in range(100):
db.insert_ts(i, ats1)
db.upsert_meta(i, {'useless': i})
db.upsert_meta(i, {'order': -i})
ids1, fields1 = db.select({'pk': {'>': 10, '<=' : 50}},None,{'limit':10,'sort_by':'-useless'})
assert(ids1 == [50, 49, 48, 47, 46, 45, 44, 43, 42, 41])
ids2, fields2 = db.select(meta={}, fields=[], additional={'limit':15,'sort_by': '-order'})
assert(ids2 == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14])
def test_insert():
db = DictDB(schema, 'pk')
t = [0,1,2,3,4]
v = [1.0,2.0,3.0,2.0,1.0]
ats = ts.TimeSeries(t, v)
db.insert_ts(1, ats)
rows = db.rows
assert(rows[1] is not None)
assert(rows[1]['pk'] == 1)
assert(rows[1]['ts'] == ats)
try:
db.insert_ts(1, ats)
except Exception as e:
e1 = e
assert str(e1) == 'Duplicate primary key found during insert'
assert type(e1).__name__ == 'ValueError'
def test_upsert():
db = DictDB(schema, 'pk')
db.insert_ts(1, ats1)
db.insert_ts(2, ats1)
db.upsert_meta(2, {'ts': ats2})
db.upsert_meta(3, {'ts': ats2, 'not_there': 3, 'order': 1})
rows = db.rows
assert (rows[1]['ts'] == ats1)
assert (rows[2]['ts'] == ats2)
assert (rows[3]['ts'] == ats2)
assert (rows[3]['order'] == 1)
def test_complex():
db = DictDB(schema, 'pk')
for i in range(100):
db.insert_ts(i, ats1)
db.upsert_meta(i, {'useless': i})
db.upsert_meta(i, {'order': -i})
ids1, fields1 = db.select({'pk': {
'>': 10,
'<=': 50
}}, None, {
'limit': 10,
'sort_by': '-useless'
})
assert (ids1 == [50, 49, 48, 47, 46, 45, 44, 43, 42, 41])
ids2, fields2 = db.select(meta={},
fields=[],
additional={
'limit': 15,
'sort_by': '-order'
})
assert (ids2 == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14])
def test_upsert():
db = DictDB(schema, 'pk')
db.insert_ts(1, ats1)
db.insert_ts(2, ats1)
db.upsert_meta(2, {'ts': ats2})
db.upsert_meta(3, {'ts': ats2, 'not_there': 3, 'order': 1})
rows = db.rows
assert(rows[1]['ts'] == ats1)
assert(rows[2]['ts'] == ats2)
assert(rows[3]['ts'] == ats2)
assert(rows[3]['order'] == 1)
def test_protocol_delete(self):
db = DictDB(schema, 'pk')
server = TSDBServer(db)
prot = TSDBProtocol(server)
t1 = [0,1,2,3,4]
v1 = [1.0,2.0,3.0,2.0,1.0]
ats1 = ts.TimeSeries(t1, v1)
t2 = [10,11,12,13,14]
v2 = [-1.0,-2.0,-3.0,-2.0,-1.0]
ats2 = ts.TimeSeries(t2, v2)
insert_op = {}
insert_op['pk'] = 1
insert_op['ts'] = ats1
insert_op['op'] = 'insert_ts'
# Test Protocol Insert
insert_return = prot._insert_ts(insert_op)
assert(insert_return['op'] == 'insert_ts')
assert(insert_return['status'] == TSDBStatus.OK)
assert(insert_return['payload'] == None)
inserted_row = server.db.rows[1]
assert(inserted_row['pk'] == 1)
assert(inserted_row['ts'] == ats1)
insert_return2 = prot._insert_ts(insert_op)
assert(insert_return2['op'] == 'insert_ts')
assert(insert_return2['status'] == TSDBStatus.INVALID_KEY)
delete_op = {}
delete_op['pk'] = 1
delete_op['op'] = 'delete_ts'
delete_return = prot._delete_ts(delete_op)
assert(delete_return['op'] == 'delete_ts')
assert(delete_return['status'] == TSDBStatus.OK)
assert(delete_return['payload'] == None)
assert (len(server.db.rows) == 0)
delete_return2 = prot._delete_ts(delete_op)
assert(delete_return2['op'] == 'delete_ts')
assert(delete_return2['status'] == TSDBStatus.INVALID_KEY)
def test_augmented_select(self):
db = DictDB(schema, 'pk')
server = TSDBServer(db)
prot = TSDBProtocol(server)
t1 = [0,1,2,3,4]
v1 = [1.0,2.0,3.0,2.0,1.0]
ats1 = ts.TimeSeries(t1, v1)
t2 = [10,11,12,13,14]
v2 = [-1.0,-2.0,-3.0,-2.0,-1.0]
ats2 = ts.TimeSeries(t2, v2)
insert_op = {}
insert_op['pk'] = 1
insert_op['ts'] = ats1
insert_op['op'] = 'insert_ts'
# Test Protocol Insert
insert_return = prot._insert_ts(insert_op)
assert(insert_return['op'] == 'insert_ts')
assert(insert_return['status'] == TSDBStatus.OK)
assert(insert_return['payload'] == None)
inserted_row = server.db.rows[1]
assert(inserted_row['pk'] == 1)
assert(inserted_row['ts'] == ats1)
# Test Protocol Select (None fields)
metadata_dict = {'pk': {'>': 0}}
fields = None
additional = None
aug_select_op = TSDBOp_AugmentedSelect('corr', ['mean', 'std'], [t2,v2], metadata_dict, additional )
aug_select_return = prot._augmented_select(aug_select_op)
assert(aug_select_return['op'] == 'augmented_select')
assert(aug_select_return['status'] == TSDBStatus.OK)
assert(aug_select_return['payload'] == {1: {'mean': 1.4142135623730403}})
def test_protocol_triggers(self):
db = DictDB(schema, 'pk')
server = TSDBServer(db)
prot = TSDBProtocol(server)
# Test Add Trigger
add_trigger_op = TSDBOp_AddTrigger('stats', 'insert_ts', ['mean', 'std'], None)
prot._add_trigger(add_trigger_op)
mod = import_module('procs.stats')
storedproc = getattr(mod,'main')
assert(server.triggers['insert_ts'] == [('stats', storedproc, None, ['mean', 'std'])])
# Test delete Trigger
delete_trigger_op = TSDBOp_RemoveTrigger('stats', 'insert_ts')
prot._remove_trigger(delete_trigger_op)
mod = import_module('procs.stats')
storedproc = getattr(mod,'main')
assert(server.triggers['insert_ts'] == [])
def test_insert():
db = DictDB(schema, 'pk')
t = [0, 1, 2, 3, 4]
v = [1.0, 2.0, 3.0, 2.0, 1.0]
ats = ts.TimeSeries(t, v)
db.insert_ts(1, ats)
rows = db.rows
assert (rows[1] is not None)
assert (rows[1]['pk'] == 1)
assert (rows[1]['ts'] == ats)
try:
db.insert_ts(1, ats)
except Exception as e:
e1 = e
assert str(e1) == 'Duplicate primary key found during insert'
assert type(e1).__name__ == 'ValueError'
class DictDBTests(unittest.TestCase):
def setUp(self):
identity = lambda x: x
to_int = lambda x:int(x)
to_float = lambda x:float(x)
to_bool = lambda x:bool(x)
schema = {
'pk': {'convert': identity, 'index': None}, #will be indexed anyways
'ts': {'convert': identity, 'index': None},
'order': {'convert': to_int, 'index': 1},
'blarg': {'convert': to_int, 'index': 1},
'useless': {'convert': identity, 'index': None},
'mean': {'convert': to_float, 'index': 1},
'std': {'convert': to_float, 'index': 1},
'vp': {'convert': to_bool, 'index': 1}
}
self.db = DictDB(schema,'pk',3)
self.db.insert_ts('one',ts.TimeSeries([1, 2, 3],[1, 4, 9]))
self.db.insert_ts('two',ts.TimeSeries([2, 3, 4],[4, 9, 16]))
self.db.insert_ts('three',ts.TimeSeries([9,3,4],[4,0,16]))
self.db.insert_ts('four',ts.TimeSeries([0,0,4],[1,0,4]))
self.db.upsert_meta('one', {'order': 1, 'blarg': 1})
self.db.upsert_meta('two', {'order': 2})
self.db.upsert_meta('three', {'order': 1, 'blarg': 2})
self.db.upsert_meta('four', {'order': 2, 'blarg': 2})
def tearDown(self):
pass
def test_insert_duplicate(self):
with self.assertRaises(ValueError):
self.db.insert_ts('two',ts.TimeSeries([2, 3, 4],[4, 9, 16]))
def test_insert_wronglen(self):
with self.assertRaises(ValueError):
self.db.insert_ts('nine',ts.TimeSeries([2, 3, 4, 5],[4, 9, 16, 25]))
def test_select1(self):
print("\n", "select1")
pks, payload = self.db.select({},None,None)
self.assertEqual(set(pks), set(['one', 'two', 'three', 'four']))
def test_select2(self):
print("\n", "select2")
pks, payload = self.db.select({'order': 1},None,None)
self.assertEqual(set(pks), set(['one', 'three']))
def test_select3(self):
print("\n", "select2")
pks, payload = self.db.select({'order': 1},None,None)
self.assertEqual(set(pks), set(['one', 'three']))
def test_select4(self):
print("\n", "select4")
pks, payload = self.db.select({'order': 1, 'blarg': 2}, ['ts'], None)
self.assertEqual(set(pks), set(['three']))
self.assertEqual(payload[0]['ts'], ts.TimeSeries([9,3,4],[4,0,16]))
def test_select5(self):
pks, payload = self.db.select({'order': {'>': 1}, 'blarg': 2}, ['pk','blarg','order'], None)
self.assertEqual(set(pks), set(['four']))
self.assertEqual(set(payload[0].keys()), set(['pk','blarg','order']))
def test_select6(self):
pks, payload = self.db.select({'order': {'>': 1}}, [], None)
self.assertEqual(set(pks), set(['two', 'four']))
for p in payload:
if p['pk'] == 'two':
self.assertEqual(set(p.keys()), set(['pk','order']))
if p['pk'] == 'four':
self.assertEqual(set(p.keys()), set(['pk','order','blarg']))
def test_select7(self):
with self.assertRaises(Exception):
pks, payload = self.db.select({'order': {'>': 1}}, [], {'sort_by':'order'})
def test_select8(self):
self.db.upsert_meta('four', {'order': 3, 'blarg': 2})
pks, payload = self.db.select({'order': {'>': 1}}, [], {'sort_by':'+order'})
self.assertEqual(pks, ['two', 'four'])
pks, payload = self.db.select({'order': {'>': 1}}, [], {'sort_by':'-order'})
self.assertEqual(pks, ['four', 'two'])
self.db.upsert_meta('four', {'order': 2, 'blarg': 2})
def test_select9(self):
self.db.upsert_meta('four', {'order': 3, 'blarg': 2})
pks, payload = self.db.select({'order': {'>': 1}}, [], {'sort_by':'+order',
'limit': 1})
self.assertEqual(pks, ['two'])
self.db.upsert_meta('four', {'order': 2, 'blarg': 2})
def test_select10(self):
self.db.upsert_meta('four', {'order': 3, 'blarg': 2})
pks, payload = self.db.select({'order': {'>': 1}}, [], {'sort_by':'-order',
'limit': 1,'blah':'nonsense'})
self.assertEqual(pks, ['four'])
self.db.upsert_meta('four', {'order': 2, 'blarg': 2})
def test_del1(self):
self.db.upsert_meta('four', {'order': 3, 'blarg': 2})
self.db.delete_ts('four')
pks, payload = self.db.select({'order': {'>': 1}}, [], {'sort_by':'-order',
'limit': 1,'blah':'nonsense'})
self.assertEqual(pks, ['two'])
self.db.insert_ts('four',ts.TimeSeries([0,0,4],[1,0,4]))
self.db.upsert_meta('four', {'order': 2, 'blarg': 2})
def test_del2(self):
self.db.delete_ts('one')
self.db.delete_ts('two')
self.db.delete_ts('three')
self.db.delete_ts('four')
pks, payload = self.db.select({},None,None)
self.assertEqual(set(pks), set([]))
self.db.insert_ts('one',ts.TimeSeries([1, 2, 3],[1, 4, 9]))
self.db.insert_ts('two',ts.TimeSeries([2, 3, 4],[4, 9, 16]))
self.db.insert_ts('three',ts.TimeSeries([9,3,4],[4,0,16]))
self.db.insert_ts('four',ts.TimeSeries([0,0,4],[1,0,4]))
self.db.upsert_meta('one', {'order': 1, 'blarg': 1})
self.db.upsert_meta('two', {'order': 2})
self.db.upsert_meta('three', {'order': 1, 'blarg': 2})
self.db.upsert_meta('four', {'order': 2, 'blarg': 2})
def test_del3(self):
with self.assertRaises(ValueError):
self.db.delete_ts('five')
def test_select11(self):
with self.assertRaises(Exception):
pks, payload = self.db.select({'order': {'>': 1}}, [], {'sort_by':'+order',
'limit':'a'})
def test_protocol(self):
db = DictDB(schema, 'pk')
server = TSDBServer(db)
prot = TSDBProtocol(server)
# Dumb server tests
assert(server.db == db)
assert(server.port == 9999)
t1 = [0,1,2,3,4]
v1 = [1.0,2.0,3.0,2.0,1.0]
ats1 = ts.TimeSeries(t1, v1)
t2 = [10,11,12,13,14]
v2 = [-1.0,-2.0,-3.0,-2.0,-1.0]
ats2 = ts.TimeSeries(t2, v2)
# Test TSDBOp_InsertTS
insert_op = {}
insert_op['pk'] = 1
insert_op['ts'] = ats1
insert_op['op'] = 'insert_ts'
InsertedTS = TSDBOp_InsertTS(1, ats1)
assert(insert_op == InsertedTS)
# Test Protocol Insert
insert_return = prot._insert_ts(insert_op)
assert(insert_return['op'] == 'insert_ts')
assert(insert_return['status'] == TSDBStatus.OK)
assert(insert_return['payload'] == None)
inserted_row = server.db.rows[1]
assert(inserted_row['pk'] == 1)
assert(inserted_row['ts'] == ats1)
# Add some more data
prot._insert_ts(TSDBOp_InsertTS(2, ats1))
inserted_row = server.db.rows[2]
assert(inserted_row['ts'] == ats1)
# Test Protocol Upsert
upserted_meta = TSDBOp_UpsertMeta(2, {'ts': ats2, 'order': 1})
upsert_return = prot._upsert_meta(upserted_meta)
assert(upsert_return['op'] == 'upsert_meta')
assert(upsert_return['status'] == TSDBStatus.OK)
assert(upsert_return['payload'] == None)
# Test Protocol Select (None fields)
metadata_dict = {'pk': {'>': 0}}
fields = None
additional = None
select_op = TSDBOp_Select(metadata_dict, fields, additional)
select_return = prot._select(select_op)
print("Here", select_return)
assert(select_return['op'] == 'select')
assert(select_return['status'] == TSDBStatus.OK)
assert(select_return['payload'][1] == {})
assert(select_return['payload'][2] == {})
# Test Protocol Select
metadata_dict = {'pk': {'>': 0}}
fields = ['ts']
additional = None
select_op = TSDBOp_Select(metadata_dict, fields, additional)
select_return = prot._select(select_op)
assert(select_return['op'] == 'select')
assert(select_return['status'] == TSDBStatus.OK)
assert(select_return['payload'][1]['ts'] == ats1)
assert(select_return['payload'][2]['ts'] == ats2)
# Test Add Trigger
add_trigger_op = TSDBOp_AddTrigger('stats', 'insert_ts', ['mean', 'std'], None)
prot._add_trigger(add_trigger_op)
mod = import_module('procs.stats')
storedproc = getattr(mod,'main')
assert(server.triggers['insert_ts'] == [('stats', storedproc, None, ['mean', 'std'])])
def test_select_basic_operations():
db = DictDB(schema, 'pk')
db.insert_ts(1, ats1)
db.insert_ts(2, ats2)
db.insert_ts(3, ats3)
ids1, fields1 = db.select({'pk': {'==': 1}},None,None)
assert(ids1 == [1])
ids2, fields2 = db.select({'pk': {'>': 1}},None,None)
assert(ids2 == [2, 3])
ids3, fields3 = db.select({'pk': {'<': 2}},None,None)
assert(ids3 == [1])
ids4, fields4 = db.select({'pk': {'!=': 1}},None,None)
assert(ids4 == [2, 3])
ids5, fields5 = db.select({'pk': {'<=': 2}},None,None)
assert(ids5 == [1, 2])
ids6, fields6 = db.select({'pk': {'>=': 2}},None,None)
assert(ids6 == [2, 3])
ids7, fields7 = db.select({'pk': {'>': 1, '<': 3}},None,None)
assert(ids7 == [2])
def test_delete():
db = DictDB(schema, 'pk')
# Upserting
db.upsert_meta(1, {'ts': ats1, 'blarg': 3, 'order': 1})
db.upsert_meta(2, {'ts': ats1, 'order': 1})
db.upsert_meta(3, {'ts': ats1, 'not_there': 3, 'order': 3})
db.upsert_meta(4, {'ts': ats1, 'blarg': 3})
# Deleting
db.delete_ts(1)
db.delete_ts(2)
db.delete_ts(3)
# Tests
rows = db.rows
assert(len(rows) == 1)
assert(4 in rows)
assert(1 not in rows)
assert(db.indexes['order'] == {})
assert(db.indexes['blarg'] == {3: {4}})
# Check to ensure that PKs that have been deleted are no
# longer in the index
assert(db.indexes['pk'] == {4: {4}})
def test_select_basic_fields():
db = DictDB(schema, 'pk')
db.insert_ts(1, ats1)
db.insert_ts(2, ats2)
db.insert_ts(3, ats3)
db.upsert_meta(2, {'useless': 2})
# One result
ids1, fields1 = db.select({'pk': {'==': 1}}, ['ts'], None)
assert (ids1 == [1])
assert (fields1[0]['ts'] == ats1)
# Two results
ids2, fields2 = db.select({'pk': {'>': 1}}, ['ts'], None)
assert (ids2 == [2, 3])
assert (fields2[0]['ts'] == ats2)
assert (fields2[1]['ts'] == ats3)
# No results
ids3, fields3 = db.select({'blarg': {'=': 1}}, ['ts'], None)
assert (ids3 == [])
# None Field List (just pks)
ids4, fields4 = db.select({'pk': {'>': 0}}, None, None)
assert (ids4 == [1, 2, 3])
assert (fields4 == [{}, {}, {}])
# Empty Field List (everything but ts)
ids5, fields5 = db.select({'pk': {'>': 0}}, [], None)
assert (ids5 == [1, 2, 3])
assert (fields5 == [{'pk': 1}, {'pk': 2, 'useless': 2}, {'pk': 3}])
# Named Field List (just that field)
ids6, fields6 = db.select({'useless': {'>': 0}}, ['useless'], None)
assert (ids6 == [2])
assert (fields6 == [{'useless': 2}])
def test_tsdb_dictdb():
# synthetic data
t = np.array([1, 1.5, 2, 2.5, 10, 11, 12])
v1 = np.array([10, 12, -11, 1.5, 10, 13, 17])
v2 = np.array([8, 12, -11, 1.5, 10, 13, 17])
a1 = TimeSeries(t, v1)
a2 = TimeSeries(t, v2)
identity = lambda x: x
schema = {
'pk': {'convert': identity, 'index': None},
'ts': {'convert': identity, 'index': None},
'order': {'convert': int, 'index': 1},
'blarg': {'convert': int, 'index': 1},
'useless': {'convert': identity, 'index': None},
'mean': {'convert': float, 'index': 1},
'std': {'convert': float, 'index': 1},
'vp': {'convert': bool, 'index': 1},
'deleted': {'convert': bool, 'index': 1}
}
# create dictionary
ddb = DictDB(schema, 'pk')
# CHECK INSERTION/UPSERTION/DELETION -->
# insert two new time series and metadata
ddb.insert_ts('pk1', a1)
ddb.upsert_meta('pk1', {'order': 1, 'blarg': 2})
ddb.insert_ts('pk2', a2)
ddb.upsert_meta('pk2', {'order': 2, 'blarg': 2})
# try to insert a duplicate primary key
with pytest.raises(ValueError):
ddb.insert_ts('pk2', a2)
# delete a valid time series
ddb.delete_ts('pk1')
print(ddb.indexes)
# check that it isn't present any more
pk, selected = ddb.select({'pk': 'pk1'}, [], None)
assert pk == []
assert len(selected) == 0
# add the time series back in
ddb.insert_ts('pk1', a1)
# Test consecutives meta upsert
ddb.upsert_meta('pk1', {'order': 2, 'blarg': 3})
for k, v in ddb.indexes['order'].items():
if k == 2:
assert ('pk1' in v)
else:
assert ('pk1' not in v)
ddb.upsert_meta('pk1', {'order': 1, 'blarg': 2})
for k, v in ddb.indexes['blarg'].items():
if k == 2:
assert ('pk1' in v)
else:
assert ('pk1' not in v)
# check that it's present now
pk, selected = ddb.select({'pk': 'pk1'}, [], None)
assert pk == ['pk1']
assert len(selected) == 1
# delete an invalid time series
with pytest.raises(ValueError):
ddb.delete_ts('not_here')
# try to insert metadata for a time series that isn't present
with pytest.raises(ValueError):
ddb.upsert_meta('pk3', {'order': 2, 'blarg': 2})
# extract database entries for testing
db_rows = ddb.rows
idx = sorted(db_rows.keys()) # sorted primary keys
# check primary keys
assert idx == ['0DELETED_pk1', 'pk1', 'pk2']
# check metadata
assert db_rows['pk1']['order'] == 1
assert db_rows['pk2']['order'] == 2
assert db_rows['pk1']['blarg'] == 2
assert db_rows['pk2']['blarg'] == 2
# CHECK SELECT OPERATIONS -->
pk, selected = ddb.select({}, None, None)
assert sorted(pk) == ['pk1', 'pk2']
assert selected == [{}, {}]
pk, selected = ddb.select({}, None, {'sort_by': '-order', 'limit': 5})
assert sorted(pk) == ['pk1', 'pk2']
assert selected == [{}, {}]
pk, selected = ddb.select({}, None, {'sort_by': '+pk'})
assert pk == ['pk1', 'pk2']
assert selected == [{}, {}]
pk, selected = ddb.select({'order': 1, 'blarg': 2}, [], None)
assert pk == ['pk1']
assert len(selected) == 1
assert selected[0]['pk'] == 'pk1'
assert selected[0]['order'] == 1
assert selected[0]['blarg'] == 2
pk, selected = ddb.select({'order': [1, 2], 'blarg': 2}, [], None)
assert sorted(pk) == ['pk1', 'pk2']
assert len(selected) == 2
idx = pk.index('pk1')
assert selected[idx]['pk'] == 'pk1'
assert selected[idx]['order'] == 1
assert selected[idx]['blarg'] == 2
pk, selected = ddb.select({'order': {'>=': 4}}, ['order'], None)
assert len(pk) == 0
assert len(selected) == 0
# field not in schema
with pytest.raises(ValueError):
ddb.select({}, None, {'sort_by': '-unknown', 'limit': 5})
# bulk update of indices
ddb.index_bulk()
check_indexes = ['blarg', 'deleted', 'mean', 'order', 'std', 'vp']
assert sorted(ddb.indexes.keys()) == check_indexes
for v in ddb.indexes.values():
assert isinstance(v, defaultdict)
def test_delete():
db = DictDB(schema, 'pk')
# Upserting
db.upsert_meta(1, {'ts': ats1, 'blarg': 3, 'order': 1})
db.upsert_meta(2, {'ts': ats1, 'order': 1})
db.upsert_meta(3, {'ts': ats1, 'not_there': 3, 'order': 3})
db.upsert_meta(4, {'ts': ats1, 'blarg': 3})
# Deleting
db.delete_ts(1)
db.delete_ts(2)
db.delete_ts(3)
# Tests
rows = db.rows
assert (len(rows) == 1)
assert (4 in rows)
assert (1 not in rows)
assert (db.indexes['order'] == {})
assert (db.indexes['blarg'] == {3: {4}})
# Check to ensure that PKs that have been deleted are no
# longer in the index
assert (db.indexes['pk'] == {4: {4}})
def test_select_basic_additional():
db = DictDB(schema, 'pk')
db.insert_ts(1, ats1)
db.insert_ts(2, ats2)
db.insert_ts(3, ats3)
db.upsert_meta(1, {'useless': 1})
db.upsert_meta(2, {'useless': 3})
db.upsert_meta(3, {'useless': 5})
# Limit to 2 results
ids1, fields1 = db.select({'pk': {'>': 0}},None,{'limit':2})
assert(ids1 == [1, 2])
# Order Ascending
ids2, fields2 = db.select({'pk': {'>': 0}},None,{'sort_by':'+useless'})
assert(ids2 == [1, 2, 3])
# Order Descending
ids3, fields3 = db.select({'pk': {'>': 0}},None,{'sort_by':'-useless'})
assert(ids3 == [3, 2, 1])
def test_select_basic_fields():
db = DictDB(schema, 'pk')
db.insert_ts(1, ats1)
db.insert_ts(2, ats2)
db.insert_ts(3, ats3)
db.upsert_meta(2, {'useless': 2})
# One result
ids1, fields1 = db.select({'pk': {'==': 1}},['ts'],None)
assert(ids1 == [1])
assert(fields1[0]['ts'] == ats1)
# Two results
ids2, fields2 = db.select({'pk': {'>': 1}},['ts'],None)
assert(ids2 == [2,3])
assert(fields2[0]['ts'] == ats2)
assert(fields2[1]['ts'] == ats3)
# No results
ids3, fields3 = db.select({'blarg': {'=': 1}},['ts'],None)
assert(ids3 == [])
# None Field List (just pks)
ids4, fields4 = db.select({'pk':{'>': 0}},None,None)
assert(ids4 == [1, 2, 3])
assert(fields4 == [{}, {}, {}])
# Empty Field List (everything but ts)
ids5, fields5 = db.select({'pk':{'>': 0}},[],None)
assert(ids5 == [1, 2, 3])
assert(fields5 == [{'pk': 1}, {'pk': 2, 'useless': 2}, {'pk': 3}])
# Named Field List (just that field)
ids6, fields6 = db.select({'useless':{'>': 0}},['useless'],None)
assert(ids6 == [2])
assert(fields6 == [{'useless': 2}])
def test_select_basic_additional():
db = DictDB(schema, 'pk')
db.insert_ts(1, ats1)
db.insert_ts(2, ats2)
db.insert_ts(3, ats3)
db.upsert_meta(1, {'useless': 1})
db.upsert_meta(2, {'useless': 3})
db.upsert_meta(3, {'useless': 5})
# Limit to 2 results
ids1, fields1 = db.select({'pk': {'>': 0}}, None, {'limit': 2})
assert (ids1 == [1, 2])
# Order Ascending
ids2, fields2 = db.select({'pk': {'>': 0}}, None, {'sort_by': '+useless'})
assert (ids2 == [1, 2, 3])
# Order Descending
ids3, fields3 = db.select({'pk': {'>': 0}}, None, {'sort_by': '-useless'})
assert (ids3 == [3, 2, 1])
class DictDBTests(unittest.TestCase):
def setUp(self):
identity = lambda x: x
to_int = lambda x: int(x)
to_float = lambda x: float(x)
to_bool = lambda x: bool(x)
schema = {
'pk': {
'convert': identity,
'index': None
}, #will be indexed anyways
'ts': {
'convert': identity,
'index': None
},
'order': {
'convert': to_int,
'index': 1
},
'blarg': {
'convert': to_int,
'index': 1
},
'useless': {
'convert': identity,
'index': None
},
'mean': {
'convert': to_float,
'index': 1
},
'std': {
'convert': to_float,
'index': 1
},
'vp': {
'convert': to_bool,
'index': 1
}
}
self.db = DictDB(schema, 'pk', 3)
self.db.insert_ts('one', ts.TimeSeries([1, 2, 3], [1, 4, 9]))
self.db.insert_ts('two', ts.TimeSeries([2, 3, 4], [4, 9, 16]))
self.db.insert_ts('three', ts.TimeSeries([9, 3, 4], [4, 0, 16]))
self.db.insert_ts('four', ts.TimeSeries([0, 0, 4], [1, 0, 4]))
self.db.upsert_meta('one', {'order': 1, 'blarg': 1})
self.db.upsert_meta('two', {'order': 2})
self.db.upsert_meta('three', {'order': 1, 'blarg': 2})
self.db.upsert_meta('four', {'order': 2, 'blarg': 2})
def tearDown(self):
pass
def test_insert_duplicate(self):
with self.assertRaises(ValueError):
self.db.insert_ts('two', ts.TimeSeries([2, 3, 4], [4, 9, 16]))
def test_insert_wronglen(self):
with self.assertRaises(ValueError):
self.db.insert_ts('nine',
ts.TimeSeries([2, 3, 4, 5], [4, 9, 16, 25]))
def test_select1(self):
print("\n", "select1")
pks, payload = self.db.select({}, None, None)
self.assertEqual(set(pks), set(['one', 'two', 'three', 'four']))
def test_select2(self):
print("\n", "select2")
pks, payload = self.db.select({'order': 1}, None, None)
self.assertEqual(set(pks), set(['one', 'three']))
def test_select3(self):
print("\n", "select2")
pks, payload = self.db.select({'order': 1}, None, None)
self.assertEqual(set(pks), set(['one', 'three']))
def test_select4(self):
print("\n", "select4")
pks, payload = self.db.select({'order': 1, 'blarg': 2}, ['ts'], None)
self.assertEqual(set(pks), set(['three']))
self.assertEqual(payload[0]['ts'], ts.TimeSeries([9, 3, 4],
[4, 0, 16]))
def test_select5(self):
pks, payload = self.db.select({
'order': {
'>': 1
},
'blarg': 2
}, ['pk', 'blarg', 'order'], None)
self.assertEqual(set(pks), set(['four']))
self.assertEqual(set(payload[0].keys()), set(['pk', 'blarg', 'order']))
def test_select6(self):
pks, payload = self.db.select({'order': {'>': 1}}, [], None)
self.assertEqual(set(pks), set(['two', 'four']))
for p in payload:
if p['pk'] == 'two':
self.assertEqual(set(p.keys()), set(['pk', 'order']))
if p['pk'] == 'four':
self.assertEqual(set(p.keys()), set(['pk', 'order', 'blarg']))
def test_select7(self):
with self.assertRaises(Exception):
pks, payload = self.db.select({'order': {
'>': 1
}}, [], {'sort_by': 'order'})
def test_select8(self):
self.db.upsert_meta('four', {'order': 3, 'blarg': 2})
pks, payload = self.db.select({'order': {
'>': 1
}}, [], {'sort_by': '+order'})
self.assertEqual(pks, ['two', 'four'])
pks, payload = self.db.select({'order': {
'>': 1
}}, [], {'sort_by': '-order'})
self.assertEqual(pks, ['four', 'two'])
self.db.upsert_meta('four', {'order': 2, 'blarg': 2})
def test_select9(self):
self.db.upsert_meta('four', {'order': 3, 'blarg': 2})
pks, payload = self.db.select({'order': {
'>': 1
}}, [], {
'sort_by': '+order',
'limit': 1
})
self.assertEqual(pks, ['two'])
self.db.upsert_meta('four', {'order': 2, 'blarg': 2})
def test_select10(self):
self.db.upsert_meta('four', {'order': 3, 'blarg': 2})
pks, payload = self.db.select({'order': {
'>': 1
}}, [], {
'sort_by': '-order',
'limit': 1,
'blah': 'nonsense'
})
self.assertEqual(pks, ['four'])
self.db.upsert_meta('four', {'order': 2, 'blarg': 2})
def test_del1(self):
self.db.upsert_meta('four', {'order': 3, 'blarg': 2})
self.db.delete_ts('four')
pks, payload = self.db.select({'order': {
'>': 1
}}, [], {
'sort_by': '-order',
'limit': 1,
'blah': 'nonsense'
})
self.assertEqual(pks, ['two'])
self.db.insert_ts('four', ts.TimeSeries([0, 0, 4], [1, 0, 4]))
self.db.upsert_meta('four', {'order': 2, 'blarg': 2})
def test_del2(self):
self.db.delete_ts('one')
self.db.delete_ts('two')
self.db.delete_ts('three')
self.db.delete_ts('four')
pks, payload = self.db.select({}, None, None)
self.assertEqual(set(pks), set([]))
self.db.insert_ts('one', ts.TimeSeries([1, 2, 3], [1, 4, 9]))
self.db.insert_ts('two', ts.TimeSeries([2, 3, 4], [4, 9, 16]))
self.db.insert_ts('three', ts.TimeSeries([9, 3, 4], [4, 0, 16]))
self.db.insert_ts('four', ts.TimeSeries([0, 0, 4], [1, 0, 4]))
self.db.upsert_meta('one', {'order': 1, 'blarg': 1})
self.db.upsert_meta('two', {'order': 2})
self.db.upsert_meta('three', {'order': 1, 'blarg': 2})
self.db.upsert_meta('four', {'order': 2, 'blarg': 2})
def test_del3(self):
with self.assertRaises(ValueError):
self.db.delete_ts('five')
def test_select11(self):
with self.assertRaises(Exception):
pks, payload = self.db.select({'order': {
'>': 1
}}, [], {
'sort_by': '+order',
'limit': 'a'
})
def test_select_basic_operations():
db = DictDB(schema, 'pk')
db.insert_ts(1, ats1)
db.insert_ts(2, ats2)
db.insert_ts(3, ats3)
ids1, fields1 = db.select({'pk': {'==': 1}}, None, None)
assert (ids1 == [1])
ids2, fields2 = db.select({'pk': {'>': 1}}, None, None)
assert (ids2 == [2, 3])
ids3, fields3 = db.select({'pk': {'<': 2}}, None, None)
assert (ids3 == [1])
ids4, fields4 = db.select({'pk': {'!=': 1}}, None, None)
assert (ids4 == [2, 3])
ids5, fields5 = db.select({'pk': {'<=': 2}}, None, None)
assert (ids5 == [1, 2])
ids6, fields6 = db.select({'pk': {'>=': 2}}, None, None)
assert (ids6 == [2, 3])
ids7, fields7 = db.select({'pk': {'>': 1, '<': 3}}, None, None)
assert (ids7 == [2])
