def test_complex(): db = DictDB(schema, 'pk') for i in range(100): db.insert_ts(i, ats1) db.upsert_meta(i, {'useless': i}) db.upsert_meta(i, {'order': -i}) ids1, fields1 = db.select({'pk': {'>': 10, '<=' : 50}},None,{'limit':10,'sort_by':'-useless'}) assert(ids1 == [50, 49, 48, 47, 46, 45, 44, 43, 42, 41]) ids2, fields2 = db.select(meta={}, fields=[], additional={'limit':15,'sort_by': '-order'}) assert(ids2 == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14])
def test_delete(): db = DictDB(schema, 'pk') # Upserting db.upsert_meta(1, {'ts': ats1, 'blarg': 3, 'order': 1}) db.upsert_meta(2, {'ts': ats1, 'order': 1}) db.upsert_meta(3, {'ts': ats1, 'not_there': 3, 'order': 3}) db.upsert_meta(4, {'ts': ats1, 'blarg': 3}) # Deleting db.delete_ts(1) db.delete_ts(2) db.delete_ts(3) # Tests rows = db.rows assert(len(rows) == 1) assert(4 in rows) assert(1 not in rows) assert(db.indexes['order'] == {}) assert(db.indexes['blarg'] == {3: {4}}) # Check to ensure that PKs that have been deleted are no # longer in the index assert(db.indexes['pk'] == {4: {4}})
def test_delete(): db = DictDB(schema, 'pk') # Upserting db.upsert_meta(1, {'ts': ats1, 'blarg': 3, 'order': 1}) db.upsert_meta(2, {'ts': ats1, 'order': 1}) db.upsert_meta(3, {'ts': ats1, 'not_there': 3, 'order': 3}) db.upsert_meta(4, {'ts': ats1, 'blarg': 3}) # Deleting db.delete_ts(1) db.delete_ts(2) db.delete_ts(3) # Tests rows = db.rows assert (len(rows) == 1) assert (4 in rows) assert (1 not in rows) assert (db.indexes['order'] == {}) assert (db.indexes['blarg'] == {3: {4}}) # Check to ensure that PKs that have been deleted are no # longer in the index assert (db.indexes['pk'] == {4: {4}})
def test_upsert(): db = DictDB(schema, 'pk') db.insert_ts(1, ats1) db.insert_ts(2, ats1) db.upsert_meta(2, {'ts': ats2}) db.upsert_meta(3, {'ts': ats2, 'not_there': 3, 'order': 1}) rows = db.rows assert(rows[1]['ts'] == ats1) assert(rows[2]['ts'] == ats2) assert(rows[3]['ts'] == ats2) assert(rows[3]['order'] == 1)
def test_upsert(): db = DictDB(schema, 'pk') db.insert_ts(1, ats1) db.insert_ts(2, ats1) db.upsert_meta(2, {'ts': ats2}) db.upsert_meta(3, {'ts': ats2, 'not_there': 3, 'order': 1}) rows = db.rows assert (rows[1]['ts'] == ats1) assert (rows[2]['ts'] == ats2) assert (rows[3]['ts'] == ats2) assert (rows[3]['order'] == 1)
def test_select_basic_additional(): db = DictDB(schema, 'pk') db.insert_ts(1, ats1) db.insert_ts(2, ats2) db.insert_ts(3, ats3) db.upsert_meta(1, {'useless': 1}) db.upsert_meta(2, {'useless': 3}) db.upsert_meta(3, {'useless': 5}) # Limit to 2 results ids1, fields1 = db.select({'pk': {'>': 0}},None,{'limit':2}) assert(ids1 == [1, 2]) # Order Ascending ids2, fields2 = db.select({'pk': {'>': 0}},None,{'sort_by':'+useless'}) assert(ids2 == [1, 2, 3]) # Order Descending ids3, fields3 = db.select({'pk': {'>': 0}},None,{'sort_by':'-useless'}) assert(ids3 == [3, 2, 1])
def test_select_basic_additional(): db = DictDB(schema, 'pk') db.insert_ts(1, ats1) db.insert_ts(2, ats2) db.insert_ts(3, ats3) db.upsert_meta(1, {'useless': 1}) db.upsert_meta(2, {'useless': 3}) db.upsert_meta(3, {'useless': 5}) # Limit to 2 results ids1, fields1 = db.select({'pk': {'>': 0}}, None, {'limit': 2}) assert (ids1 == [1, 2]) # Order Ascending ids2, fields2 = db.select({'pk': {'>': 0}}, None, {'sort_by': '+useless'}) assert (ids2 == [1, 2, 3]) # Order Descending ids3, fields3 = db.select({'pk': {'>': 0}}, None, {'sort_by': '-useless'}) assert (ids3 == [3, 2, 1])
def test_select_basic_fields(): db = DictDB(schema, 'pk') db.insert_ts(1, ats1) db.insert_ts(2, ats2) db.insert_ts(3, ats3) db.upsert_meta(2, {'useless': 2}) # One result ids1, fields1 = db.select({'pk': {'==': 1}},['ts'],None) assert(ids1 == [1]) assert(fields1[0]['ts'] == ats1) # Two results ids2, fields2 = db.select({'pk': {'>': 1}},['ts'],None) assert(ids2 == [2,3]) assert(fields2[0]['ts'] == ats2) assert(fields2[1]['ts'] == ats3) # No results ids3, fields3 = db.select({'blarg': {'=': 1}},['ts'],None) assert(ids3 == []) # None Field List (just pks) ids4, fields4 = db.select({'pk':{'>': 0}},None,None) assert(ids4 == [1, 2, 3]) assert(fields4 == [{}, {}, {}]) # Empty Field List (everything but ts) ids5, fields5 = db.select({'pk':{'>': 0}},[],None) assert(ids5 == [1, 2, 3]) assert(fields5 == [{'pk': 1}, {'pk': 2, 'useless': 2}, {'pk': 3}]) # Named Field List (just that field) ids6, fields6 = db.select({'useless':{'>': 0}},['useless'],None) assert(ids6 == [2]) assert(fields6 == [{'useless': 2}])
def test_select_basic_fields(): db = DictDB(schema, 'pk') db.insert_ts(1, ats1) db.insert_ts(2, ats2) db.insert_ts(3, ats3) db.upsert_meta(2, {'useless': 2}) # One result ids1, fields1 = db.select({'pk': {'==': 1}}, ['ts'], None) assert (ids1 == [1]) assert (fields1[0]['ts'] == ats1) # Two results ids2, fields2 = db.select({'pk': {'>': 1}}, ['ts'], None) assert (ids2 == [2, 3]) assert (fields2[0]['ts'] == ats2) assert (fields2[1]['ts'] == ats3) # No results ids3, fields3 = db.select({'blarg': {'=': 1}}, ['ts'], None) assert (ids3 == []) # None Field List (just pks) ids4, fields4 = db.select({'pk': {'>': 0}}, None, None) assert (ids4 == [1, 2, 3]) assert (fields4 == [{}, {}, {}]) # Empty Field List (everything but ts) ids5, fields5 = db.select({'pk': {'>': 0}}, [], None) assert (ids5 == [1, 2, 3]) assert (fields5 == [{'pk': 1}, {'pk': 2, 'useless': 2}, {'pk': 3}]) # Named Field List (just that field) ids6, fields6 = db.select({'useless': {'>': 0}}, ['useless'], None) assert (ids6 == [2]) assert (fields6 == [{'useless': 2}])
def test_complex(): db = DictDB(schema, 'pk') for i in range(100): db.insert_ts(i, ats1) db.upsert_meta(i, {'useless': i}) db.upsert_meta(i, {'order': -i}) ids1, fields1 = db.select({'pk': { '>': 10, '<=': 50 }}, None, { 'limit': 10, 'sort_by': '-useless' }) assert (ids1 == [50, 49, 48, 47, 46, 45, 44, 43, 42, 41]) ids2, fields2 = db.select(meta={}, fields=[], additional={ 'limit': 15, 'sort_by': '-order' }) assert (ids2 == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14])
class DictDBTests(unittest.TestCase): def setUp(self): identity = lambda x: x to_int = lambda x:int(x) to_float = lambda x:float(x) to_bool = lambda x:bool(x) schema = { 'pk': {'convert': identity, 'index': None}, #will be indexed anyways 'ts': {'convert': identity, 'index': None}, 'order': {'convert': to_int, 'index': 1}, 'blarg': {'convert': to_int, 'index': 1}, 'useless': {'convert': identity, 'index': None}, 'mean': {'convert': to_float, 'index': 1}, 'std': {'convert': to_float, 'index': 1}, 'vp': {'convert': to_bool, 'index': 1} } self.db = DictDB(schema,'pk',3) self.db.insert_ts('one',ts.TimeSeries([1, 2, 3],[1, 4, 9])) self.db.insert_ts('two',ts.TimeSeries([2, 3, 4],[4, 9, 16])) self.db.insert_ts('three',ts.TimeSeries([9,3,4],[4,0,16])) self.db.insert_ts('four',ts.TimeSeries([0,0,4],[1,0,4])) self.db.upsert_meta('one', {'order': 1, 'blarg': 1}) self.db.upsert_meta('two', {'order': 2}) self.db.upsert_meta('three', {'order': 1, 'blarg': 2}) self.db.upsert_meta('four', {'order': 2, 'blarg': 2}) def tearDown(self): pass def test_insert_duplicate(self): with self.assertRaises(ValueError): self.db.insert_ts('two',ts.TimeSeries([2, 3, 4],[4, 9, 16])) def test_insert_wronglen(self): with self.assertRaises(ValueError): self.db.insert_ts('nine',ts.TimeSeries([2, 3, 4, 5],[4, 9, 16, 25])) def test_select1(self): print("\n", "select1") pks, payload = self.db.select({},None,None) self.assertEqual(set(pks), set(['one', 'two', 'three', 'four'])) def test_select2(self): print("\n", "select2") pks, payload = self.db.select({'order': 1},None,None) self.assertEqual(set(pks), set(['one', 'three'])) def test_select3(self): print("\n", "select2") pks, payload = self.db.select({'order': 1},None,None) self.assertEqual(set(pks), set(['one', 'three'])) def test_select4(self): print("\n", "select4") pks, payload = self.db.select({'order': 1, 'blarg': 2}, ['ts'], None) self.assertEqual(set(pks), set(['three'])) self.assertEqual(payload[0]['ts'], ts.TimeSeries([9,3,4],[4,0,16])) def test_select5(self): pks, payload = self.db.select({'order': {'>': 1}, 'blarg': 2}, ['pk','blarg','order'], None) self.assertEqual(set(pks), set(['four'])) self.assertEqual(set(payload[0].keys()), set(['pk','blarg','order'])) def test_select6(self): pks, payload = self.db.select({'order': {'>': 1}}, [], None) self.assertEqual(set(pks), set(['two', 'four'])) for p in payload: if p['pk'] == 'two': self.assertEqual(set(p.keys()), set(['pk','order'])) if p['pk'] == 'four': self.assertEqual(set(p.keys()), set(['pk','order','blarg'])) def test_select7(self): with self.assertRaises(Exception): pks, payload = self.db.select({'order': {'>': 1}}, [], {'sort_by':'order'}) def test_select8(self): self.db.upsert_meta('four', {'order': 3, 'blarg': 2}) pks, payload = self.db.select({'order': {'>': 1}}, [], {'sort_by':'+order'}) self.assertEqual(pks, ['two', 'four']) pks, payload = self.db.select({'order': {'>': 1}}, [], {'sort_by':'-order'}) self.assertEqual(pks, ['four', 'two']) self.db.upsert_meta('four', {'order': 2, 'blarg': 2}) def test_select9(self): self.db.upsert_meta('four', {'order': 3, 'blarg': 2}) pks, payload = self.db.select({'order': {'>': 1}}, [], {'sort_by':'+order', 'limit': 1}) self.assertEqual(pks, ['two']) self.db.upsert_meta('four', {'order': 2, 'blarg': 2}) def test_select10(self): self.db.upsert_meta('four', {'order': 3, 'blarg': 2}) pks, payload = self.db.select({'order': {'>': 1}}, [], {'sort_by':'-order', 'limit': 1,'blah':'nonsense'}) self.assertEqual(pks, ['four']) self.db.upsert_meta('four', {'order': 2, 'blarg': 2}) def test_del1(self): self.db.upsert_meta('four', {'order': 3, 'blarg': 2}) self.db.delete_ts('four') pks, payload = self.db.select({'order': {'>': 1}}, [], {'sort_by':'-order', 'limit': 1,'blah':'nonsense'}) self.assertEqual(pks, ['two']) self.db.insert_ts('four',ts.TimeSeries([0,0,4],[1,0,4])) self.db.upsert_meta('four', {'order': 2, 'blarg': 2}) def test_del2(self): self.db.delete_ts('one') self.db.delete_ts('two') self.db.delete_ts('three') self.db.delete_ts('four') pks, payload = self.db.select({},None,None) self.assertEqual(set(pks), set([])) self.db.insert_ts('one',ts.TimeSeries([1, 2, 3],[1, 4, 9])) self.db.insert_ts('two',ts.TimeSeries([2, 3, 4],[4, 9, 16])) self.db.insert_ts('three',ts.TimeSeries([9,3,4],[4,0,16])) self.db.insert_ts('four',ts.TimeSeries([0,0,4],[1,0,4])) self.db.upsert_meta('one', {'order': 1, 'blarg': 1}) self.db.upsert_meta('two', {'order': 2}) self.db.upsert_meta('three', {'order': 1, 'blarg': 2}) self.db.upsert_meta('four', {'order': 2, 'blarg': 2}) def test_del3(self): with self.assertRaises(ValueError): self.db.delete_ts('five') def test_select11(self): with self.assertRaises(Exception): pks, payload = self.db.select({'order': {'>': 1}}, [], {'sort_by':'+order', 'limit':'a'})
def test_tsdb_dictdb(): # synthetic data t = np.array([1, 1.5, 2, 2.5, 10, 11, 12]) v1 = np.array([10, 12, -11, 1.5, 10, 13, 17]) v2 = np.array([8, 12, -11, 1.5, 10, 13, 17]) a1 = TimeSeries(t, v1) a2 = TimeSeries(t, v2) identity = lambda x: x schema = { 'pk': {'convert': identity, 'index': None}, 'ts': {'convert': identity, 'index': None}, 'order': {'convert': int, 'index': 1}, 'blarg': {'convert': int, 'index': 1}, 'useless': {'convert': identity, 'index': None}, 'mean': {'convert': float, 'index': 1}, 'std': {'convert': float, 'index': 1}, 'vp': {'convert': bool, 'index': 1}, 'deleted': {'convert': bool, 'index': 1} } # create dictionary ddb = DictDB(schema, 'pk') # CHECK INSERTION/UPSERTION/DELETION --> # insert two new time series and metadata ddb.insert_ts('pk1', a1) ddb.upsert_meta('pk1', {'order': 1, 'blarg': 2}) ddb.insert_ts('pk2', a2) ddb.upsert_meta('pk2', {'order': 2, 'blarg': 2}) # try to insert a duplicate primary key with pytest.raises(ValueError): ddb.insert_ts('pk2', a2) # delete a valid time series ddb.delete_ts('pk1') print(ddb.indexes) # check that it isn't present any more pk, selected = ddb.select({'pk': 'pk1'}, [], None) assert pk == [] assert len(selected) == 0 # add the time series back in ddb.insert_ts('pk1', a1) # Test consecutives meta upsert ddb.upsert_meta('pk1', {'order': 2, 'blarg': 3}) for k, v in ddb.indexes['order'].items(): if k == 2: assert ('pk1' in v) else: assert ('pk1' not in v) ddb.upsert_meta('pk1', {'order': 1, 'blarg': 2}) for k, v in ddb.indexes['blarg'].items(): if k == 2: assert ('pk1' in v) else: assert ('pk1' not in v) # check that it's present now pk, selected = ddb.select({'pk': 'pk1'}, [], None) assert pk == ['pk1'] assert len(selected) == 1 # delete an invalid time series with pytest.raises(ValueError): ddb.delete_ts('not_here') # try to insert metadata for a time series that isn't present with pytest.raises(ValueError): ddb.upsert_meta('pk3', {'order': 2, 'blarg': 2}) # extract database entries for testing db_rows = ddb.rows idx = sorted(db_rows.keys()) # sorted primary keys # check primary keys assert idx == ['0DELETED_pk1', 'pk1', 'pk2'] # check metadata assert db_rows['pk1']['order'] == 1 assert db_rows['pk2']['order'] == 2 assert db_rows['pk1']['blarg'] == 2 assert db_rows['pk2']['blarg'] == 2 # CHECK SELECT OPERATIONS --> pk, selected = ddb.select({}, None, None) assert sorted(pk) == ['pk1', 'pk2'] assert selected == [{}, {}] pk, selected = ddb.select({}, None, {'sort_by': '-order', 'limit': 5}) assert sorted(pk) == ['pk1', 'pk2'] assert selected == [{}, {}] pk, selected = ddb.select({}, None, {'sort_by': '+pk'}) assert pk == ['pk1', 'pk2'] assert selected == [{}, {}] pk, selected = ddb.select({'order': 1, 'blarg': 2}, [], None) assert pk == ['pk1'] assert len(selected) == 1 assert selected[0]['pk'] == 'pk1' assert selected[0]['order'] == 1 assert selected[0]['blarg'] == 2 pk, selected = ddb.select({'order': [1, 2], 'blarg': 2}, [], None) assert sorted(pk) == ['pk1', 'pk2'] assert len(selected) == 2 idx = pk.index('pk1') assert selected[idx]['pk'] == 'pk1' assert selected[idx]['order'] == 1 assert selected[idx]['blarg'] == 2 pk, selected = ddb.select({'order': {'>=': 4}}, ['order'], None) assert len(pk) == 0 assert len(selected) == 0 # field not in schema with pytest.raises(ValueError): ddb.select({}, None, {'sort_by': '-unknown', 'limit': 5}) # bulk update of indices ddb.index_bulk() check_indexes = ['blarg', 'deleted', 'mean', 'order', 'std', 'vp'] assert sorted(ddb.indexes.keys()) == check_indexes for v in ddb.indexes.values(): assert isinstance(v, defaultdict)
class DictDBTests(unittest.TestCase): def setUp(self): identity = lambda x: x to_int = lambda x: int(x) to_float = lambda x: float(x) to_bool = lambda x: bool(x) schema = { 'pk': { 'convert': identity, 'index': None }, #will be indexed anyways 'ts': { 'convert': identity, 'index': None }, 'order': { 'convert': to_int, 'index': 1 }, 'blarg': { 'convert': to_int, 'index': 1 }, 'useless': { 'convert': identity, 'index': None }, 'mean': { 'convert': to_float, 'index': 1 }, 'std': { 'convert': to_float, 'index': 1 }, 'vp': { 'convert': to_bool, 'index': 1 } } self.db = DictDB(schema, 'pk', 3) self.db.insert_ts('one', ts.TimeSeries([1, 2, 3], [1, 4, 9])) self.db.insert_ts('two', ts.TimeSeries([2, 3, 4], [4, 9, 16])) self.db.insert_ts('three', ts.TimeSeries([9, 3, 4], [4, 0, 16])) self.db.insert_ts('four', ts.TimeSeries([0, 0, 4], [1, 0, 4])) self.db.upsert_meta('one', {'order': 1, 'blarg': 1}) self.db.upsert_meta('two', {'order': 2}) self.db.upsert_meta('three', {'order': 1, 'blarg': 2}) self.db.upsert_meta('four', {'order': 2, 'blarg': 2}) def tearDown(self): pass def test_insert_duplicate(self): with self.assertRaises(ValueError): self.db.insert_ts('two', ts.TimeSeries([2, 3, 4], [4, 9, 16])) def test_insert_wronglen(self): with self.assertRaises(ValueError): self.db.insert_ts('nine', ts.TimeSeries([2, 3, 4, 5], [4, 9, 16, 25])) def test_select1(self): print("\n", "select1") pks, payload = self.db.select({}, None, None) self.assertEqual(set(pks), set(['one', 'two', 'three', 'four'])) def test_select2(self): print("\n", "select2") pks, payload = self.db.select({'order': 1}, None, None) self.assertEqual(set(pks), set(['one', 'three'])) def test_select3(self): print("\n", "select2") pks, payload = self.db.select({'order': 1}, None, None) self.assertEqual(set(pks), set(['one', 'three'])) def test_select4(self): print("\n", "select4") pks, payload = self.db.select({'order': 1, 'blarg': 2}, ['ts'], None) self.assertEqual(set(pks), set(['three'])) self.assertEqual(payload[0]['ts'], ts.TimeSeries([9, 3, 4], [4, 0, 16])) def test_select5(self): pks, payload = self.db.select({ 'order': { '>': 1 }, 'blarg': 2 }, ['pk', 'blarg', 'order'], None) self.assertEqual(set(pks), set(['four'])) self.assertEqual(set(payload[0].keys()), set(['pk', 'blarg', 'order'])) def test_select6(self): pks, payload = self.db.select({'order': {'>': 1}}, [], None) self.assertEqual(set(pks), set(['two', 'four'])) for p in payload: if p['pk'] == 'two': self.assertEqual(set(p.keys()), set(['pk', 'order'])) if p['pk'] == 'four': self.assertEqual(set(p.keys()), set(['pk', 'order', 'blarg'])) def test_select7(self): with self.assertRaises(Exception): pks, payload = self.db.select({'order': { '>': 1 }}, [], {'sort_by': 'order'}) def test_select8(self): self.db.upsert_meta('four', {'order': 3, 'blarg': 2}) pks, payload = self.db.select({'order': { '>': 1 }}, [], {'sort_by': '+order'}) self.assertEqual(pks, ['two', 'four']) pks, payload = self.db.select({'order': { '>': 1 }}, [], {'sort_by': '-order'}) self.assertEqual(pks, ['four', 'two']) self.db.upsert_meta('four', {'order': 2, 'blarg': 2}) def test_select9(self): self.db.upsert_meta('four', {'order': 3, 'blarg': 2}) pks, payload = self.db.select({'order': { '>': 1 }}, [], { 'sort_by': '+order', 'limit': 1 }) self.assertEqual(pks, ['two']) self.db.upsert_meta('four', {'order': 2, 'blarg': 2}) def test_select10(self): self.db.upsert_meta('four', {'order': 3, 'blarg': 2}) pks, payload = self.db.select({'order': { '>': 1 }}, [], { 'sort_by': '-order', 'limit': 1, 'blah': 'nonsense' }) self.assertEqual(pks, ['four']) self.db.upsert_meta('four', {'order': 2, 'blarg': 2}) def test_del1(self): self.db.upsert_meta('four', {'order': 3, 'blarg': 2}) self.db.delete_ts('four') pks, payload = self.db.select({'order': { '>': 1 }}, [], { 'sort_by': '-order', 'limit': 1, 'blah': 'nonsense' }) self.assertEqual(pks, ['two']) self.db.insert_ts('four', ts.TimeSeries([0, 0, 4], [1, 0, 4])) self.db.upsert_meta('four', {'order': 2, 'blarg': 2}) def test_del2(self): self.db.delete_ts('one') self.db.delete_ts('two') self.db.delete_ts('three') self.db.delete_ts('four') pks, payload = self.db.select({}, None, None) self.assertEqual(set(pks), set([])) self.db.insert_ts('one', ts.TimeSeries([1, 2, 3], [1, 4, 9])) self.db.insert_ts('two', ts.TimeSeries([2, 3, 4], [4, 9, 16])) self.db.insert_ts('three', ts.TimeSeries([9, 3, 4], [4, 0, 16])) self.db.insert_ts('four', ts.TimeSeries([0, 0, 4], [1, 0, 4])) self.db.upsert_meta('one', {'order': 1, 'blarg': 1}) self.db.upsert_meta('two', {'order': 2}) self.db.upsert_meta('three', {'order': 1, 'blarg': 2}) self.db.upsert_meta('four', {'order': 2, 'blarg': 2}) def test_del3(self): with self.assertRaises(ValueError): self.db.delete_ts('five') def test_select11(self): with self.assertRaises(Exception): pks, payload = self.db.select({'order': { '>': 1 }}, [], { 'sort_by': '+order', 'limit': 'a' })